{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys, os, re\n",
    "import json\n",
    "import datetime, iso8601\n",
    "\n",
    "base_path = \"..\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup PySpark Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PySpark initiated...\n"
     ]
    }
   ],
   "source": [
    "APP_NAME = \"Debugging Prediction Problems\"\n",
    "\n",
    "# If there is no SparkSession, create the environment\n",
    "try:\n",
    "  sc and spark\n",
    "except NameError as e:\n",
    "  import findspark\n",
    "  findspark.init()\n",
    "  import pyspark\n",
    "  import pyspark.sql\n",
    "\n",
    "  sc = pyspark.SparkContext()\n",
    "  spark = pyspark.sql.SparkSession(sc).builder.appName(APP_NAME).getOrCreate()\n",
    "\n",
    "print(\"PySpark initiated...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Training DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+--------+------+----+----------+---------+--------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DepDelay|Origin|Dest|FlightDate|FlightNum|Distance|\n",
      "+--------+--------------------+--------------------+-------+--------+------+----+----------+---------+--------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|    14.0|   ABQ| DFW|2014-12-31|     1024|   569.0|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|    14.0|   ABQ| DFW|2014-12-31|     1184|   569.0|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|    -2.0|   ABQ| DFW|2014-12-31|      336|   569.0|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|    -1.0|   ATL| DFW|2014-12-31|      125|   731.0|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|    -4.0|   ATL| DFW|2014-12-31|     1455|   731.0|\n",
      "|    16.0|2015-01-01 07:15:...|2015-01-01 05:45:...|     AA|    15.0|   ATL| DFW|2014-12-31|     1473|   731.0|\n",
      "|    -7.0|2015-01-01 04:15:...|2015-01-01 02:45:...|     AA|    -2.0|   ATL| DFW|2014-12-31|     1513|   731.0|\n",
      "|    13.0|2015-01-01 08:50:...|2015-01-01 07:25:...|     AA|     9.0|   ATL| DFW|2014-12-31|      194|   731.0|\n",
      "|    25.0|2015-01-01 12:30:...|2015-01-01 11:00:...|     AA|    -2.0|   ATL| DFW|2014-12-31|      232|   731.0|\n",
      "|    58.0|2015-01-01 13:40:...|2015-01-01 12:15:...|     AA|    14.0|   ATL| DFW|2014-12-31|      276|   731.0|\n",
      "|    14.0|2015-01-01 05:25:...|2015-01-01 03:55:...|     AA|    15.0|   ATL| DFW|2014-12-31|      314|   731.0|\n",
      "|     1.0|2015-01-01 10:05:...|2015-01-01 08:40:...|     AA|    -5.0|   ATL| DFW|2014-12-31|      356|   731.0|\n",
      "+--------+--------------------+--------------------+-------+--------+------+----+----------+---------+--------+\n",
      "only showing top 12 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql.types import StringType, IntegerType, FloatType, DoubleType, DateType, TimestampType\n",
    "from pyspark.sql.types import StructType, StructField\n",
    "from pyspark.sql.functions import udf\n",
    "\n",
    "schema = StructType([\n",
    "  StructField(\"ArrDelay\", DoubleType(), True),  # \"ArrDelay\":5.0\n",
    "  StructField(\"CRSArrTime\", TimestampType(), True),  # \"CRSArrTime\":\"2015-12-31T03:20:00.000-08:00\"\n",
    "  StructField(\"CRSDepTime\", TimestampType(), True),  # \"CRSDepTime\":\"2015-12-31T03:05:00.000-08:00\"\n",
    "  StructField(\"Carrier\", StringType(), True),  # \"Carrier\":\"WN\"\n",
    "  StructField(\"DayOfMonth\", IntegerType(), True),  # \"DayOfMonth\":31\n",
    "  StructField(\"DayOfWeek\", IntegerType(), True),  # \"DayOfWeek\":4\n",
    "  StructField(\"DayOfYear\", IntegerType(), True),  # \"DayOfYear\":365\n",
    "  StructField(\"DepDelay\", DoubleType(), True),  # \"DepDelay\":14.0\n",
    "  StructField(\"Dest\", StringType(), True),  # \"Dest\":\"SAN\"\n",
    "  StructField(\"Distance\", DoubleType(), True),  # \"Distance\":368.0\n",
    "  StructField(\"FlightDate\", DateType(), True),  # \"FlightDate\":\"2015-12-30T16:00:00.000-08:00\"\n",
    "  StructField(\"FlightNum\", StringType(), True),  # \"FlightNum\":\"6109\"\n",
    "  StructField(\"Origin\", StringType(), True),  # \"Origin\":\"TUS\"\n",
    "])\n",
    "\n",
    "input_path = \"{}/data/simple_flight_delay_features.json\".format(base_path)\n",
    "\n",
    "features = spark.read.json(\n",
    "  input_path,\n",
    "  schema=schema\n",
    ")\n",
    "features.select(\"ArrDelay\", \"CRSArrTime\", \"CRSDepTime\", \"Carrier\", \"DepDelay\", \n",
    "                \"Origin\", \"Dest\", \"FlightDate\", \"FlightNum\", \"Distance\").show(12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Reproduce the ArrDelay Bucketing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------+\n",
      "|ArrDelay|ArrDelayBucket|\n",
      "+--------+--------------+\n",
      "|    13.0|           0.0|\n",
      "|    17.0|           1.0|\n",
      "|    36.0|           1.0|\n",
      "|   -21.0|           0.0|\n",
      "|   -14.0|           0.0|\n",
      "|    16.0|           1.0|\n",
      "|    -7.0|           0.0|\n",
      "|    13.0|           0.0|\n",
      "|    25.0|           1.0|\n",
      "|    58.0|           1.0|\n",
      "+--------+--------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Load the arrival delay bucketizer\n",
    "from pyspark.ml.feature import Bucketizer\n",
    "\n",
    "# Load the departure delay bucketizer\n",
    "arrival_bucketizer_path = \"{}/models/arrival_bucketizer.bin\".format(base_path)\n",
    "arrival_bucketizer = Bucketizer.load(arrival_bucketizer_path)\n",
    "\n",
    "# Bucketize the departure and arrival delays for classification\n",
    "ml_bucketized_features = arrival_bucketizer.transform(features)\n",
    "ml_bucketized_features.select(\"ArrDelay\", \"ArrDelayBucket\").show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check the `ArrDelayBucket` Distribution\n",
    "Lets start by checking the overall range of flight delays."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-------------+\n",
      "|min(ArrDelay)|max(ArrDelay)|\n",
      "+-------------+-------------+\n",
      "|        -87.0|       1971.0|\n",
      "+-------------+-------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Check the frequency of each category\n",
    "ml_bucketized_features.registerTempTable(\"ml_bucketized_features\")\n",
    "\n",
    "spark.sql(\"SELECT MIN(ArrDelay), MAX(ArrDelay) FROM ml_bucketized_features\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now lets check the distribution of arrival delays as they fall into our bucketing scheme."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-------+---------+\n",
      "|ArrDelayBucket|  Total|Total_Pct|\n",
      "+--------------+-------+---------+\n",
      "|           0.0|4650569|    81.39|\n",
      "|           1.0| 737848|    12.91|\n",
      "|           2.0| 325591|      5.7|\n",
      "+--------------+-------+---------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\n",
    "\"\"\"\n",
    "SELECT\n",
    "  ArrDelayBucket,\n",
    "  COUNT(*) AS Total,\n",
    "  ROUND(\n",
    "    100 * (COUNT(*)/(SELECT COUNT(*) FROM ml_bucketized_features)),\n",
    "    2\n",
    "  ) AS Total_Pct\n",
    "FROM ml_bucketized_features\n",
    "GROUP BY ArrDelayBucket\n",
    "\"\"\"\n",
    ").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The results are pretty skewed, with 81.4% in `ArrDelayBucket` `0.0`. Note that our model always predicts `0.0`..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sampling `ArrDelayBucket` `0.0`\n",
    "Lets take a look at some example values in the largest bucket, bucket `0.0`. Lets sample it before we inspect it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------+\n",
      "|ArrDelay|ArrDelayBucket|\n",
      "+--------+--------------+\n",
      "|   -31.0|           0.0|\n",
      "|   -26.0|           0.0|\n",
      "|    -7.0|           0.0|\n",
      "|    -9.0|           0.0|\n",
      "|    -7.0|           0.0|\n",
      "|     6.0|           0.0|\n",
      "|     0.0|           0.0|\n",
      "|    -2.0|           0.0|\n",
      "|   -15.0|           0.0|\n",
      "|   -21.0|           0.0|\n",
      "|    -5.0|           0.0|\n",
      "|    10.0|           0.0|\n",
      "|   -13.0|           0.0|\n",
      "|   -21.0|           0.0|\n",
      "|   -21.0|           0.0|\n",
      "+--------+--------------+\n",
      "only showing top 15 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Check out some values from the field\n",
    "spark.sql(\"SELECT ArrDelay, ArrDelayBucket \"\n",
    "          \"FROM ml_bucketized_features \"\n",
    "          \"WHERE ArrDelayBucket == 0.0\")\\\n",
    "  .sample(False, 0.01, 11).show(15)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There looks to be a pretty broad range of values in `ArrDelayBucket` `0.0`..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computing `ArrDelayBucket` Scope\n",
    "Lets look at the `MIN` and `MAX` values of `ArrDelay` in `ArrDelayBucket` `0.0`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-------------+\n",
      "|min(ArrDelay)|max(ArrDelay)|\n",
      "+-------------+-------------+\n",
      "|        -87.0|         14.0|\n",
      "+-------------+-------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Check out the min/max for the field\n",
    "spark.sql(\"SELECT MIN(ArrDelay), MAX(ArrDelay) \"\n",
    "          \"FROM ml_bucketized_features \"\n",
    "          \"WHERE ArrDelayBucket == 0.0\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There is a range of 101 in this bucket. This encodes a large range... from 14 minute late all the way to over an hour early."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline Histogram of `ArrDelay`\n",
    "Now, what is the distribution of values in `ArrDelayBucket` `0.0`? I want to get a sense of what this bucket encodes. We can filter to a relation with only `ArrDelayBucket` values of `0.0`, convert our `DataFrame` to an `RDD` and then use the [`RDD.histogram()`](http://spark.apache.org/docs/latest/api/python/pyspark.html#pyspark.RDD.histogram) function to get the raw data for our histogram. \n",
    "\n",
    "First, however, I need the histogram of the entire range of data to compare this to. We need to supply our own buckets, or the histogram is hard to interpret."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([-87.0, -60, -30, -15, 0, 15, 30, 60, 120],\n",
       " [392, 108707, 989113, 2402687, 1149670, 412835, 325013, 211171])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Look at overall histogram\n",
    "ml_bucketized_features\\\n",
    "  .select(\"ArrDelay\")\\\n",
    "  .rdd\\\n",
    "  .flatMap(lambda x: x)\\\n",
    "  .histogram([-87.0, -60, -30, -15, 0, 15, 30, 60, 120])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating a PyPlot Bar Chart\n",
    "We must use [matplotlib.pyplot.bar](http://matplotlib.org/api/pyplot_api.html#matplotlib.pyplot.bar) instead of [matplotlib.pyplot.hist](http://matplotlib.org/api/pyplot_api.html#matplotlib.pyplot.hist) because we have already computed our bins and their weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAFkCAYAAADYL8pXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuQX2Wd5/H3h/vImCBm7OhKRJeZGF1wSeQ2asTJCIMw\nzuxqqcGUeCvHG1KxBHcsHbKwVQquBEfwUgI6ivQUi+voQjQIpWZEJGuCsyAh7GggIibSA3bYQLjl\n2T/Oaefwm06nO+lL+sn7VXWq+Z3n2+c8v6ebX3/ynFtKKUiSJE13+0x1ByRJksaDoUaSJFXBUCNJ\nkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVWFMoSbJXydZnWRLks1J\nvpHkj3pqvpRke8+yoqfmwCSXJhlI8lCSa5I8q6fmGUm+lmQwyYNJLktycE/NYUmuS7I1yaYkFybZ\np6fmqCSrkjyS5J4kZw/zvk5MsibJtiR3JTljLOMiSZKm3lhnal4BfAY4DvhTYH/g+iS/11P3baAP\nmN0ui3vaLwZOBV4HLASeA3y9p+YqYB6wqK1dCHxhqLENLyuA/YDjgTOAtwLndWqeDqwENgDzgbOB\nZUne2ak5HLgWuBF4CfBp4LIkrx7FeEiSpD1EdueBlklmAb8BFpZSftiu+xIws5Tyn3fwPTOA+4E3\nlVK+0a6bC6wDji+lrE4yD/gZsKCUcmtbczJwHfDcUsqmJKcA3wKeXUoZaGv+CvgE8AellCeSvAc4\nH5hdSnmirfk48BellBe1ry8ATimlHNXpY3/7Hl6zy4MjSZIm1e6eU3MIUIAHetaf2B6eujPJZ5Mc\n2mlbQDO7cuPQilLKemAjcEK76njgwaFA07qh3ddxnZrbhgJNayUwE3hxp2bVUKDp1MxNMrNTc0NP\n/1d2+iJJkqaB/Xb1G5OE5jDSD0spd3Savk1zKGkD8O+BjwMrkpxQmmmh2cBjpZQtPZvc3LbRfv1N\nt7GU8mSSB3pqNg+zjaG2f2q//mKEmsERtjMjyYGllEeHee/PBE4G7ga29bZLkqQdOgg4HFhZSvmX\n8dzwLoca4LPAi4CXdVeWUq7uvPxZktuAnwMnAt/bjf2Np+zm958MfG08OiJJ0l7qzTTnz46bXQo1\nSS4BXgO8opTy65FqSykbkgwAR9CEmk3AAUlm9MzW9LVttF97r4baFzi0p+aYnt31ddqGvvYNU1NG\nUbNluFma1t0AV155JfPmzdtBicbb0qVLWb58+VR3Y6/imE8+x3zyOeaTa926dSxZsgTav6Xjacyh\npg00fwG8spSycRT1zwWeCQyFnzXAEzRXNXVPFJ4D3NzW3AwckuToznk1i2hmWG7p1HwkyazOeTUn\n0RxSuqNT89+S7FtKebJTs76UMtipOaWn2yd1+jKcbQDz5s1j/vz5OxkBjZeZM2c63pPMMZ98jvnk\nc8ynzLifvjHW+9R8lma66HRga5K+djmobT+4vVfMcUmel2QR8A/AXTQn39LOzlwOXNTeH2YBcAVw\nUylldVtzZ1v/xSTHJHkZzaXk/aWUoRmW62nCy1fbe9GcTHOl0yWllMfbmquAx4ArkrwoyRuBDwCf\n6rytzwMvSHJBkrlJ3gu8HrhoLGMjSZKm1livfno3MAP4PnBfZ3lD2/4kcBTwTWA98EXgf9Nc8v14\nZztLae4Nc01nW6/r2dfpwJ00VyZdC6wC/mqosZSyHTit3eePgK8AXwbO7dRsoZl1ORz4CfBJYFkp\n5fJOzd0098H5U+Cnbd/eUUrpvSJKkiTtwcZ0+KmUMmIIKqVsA/5sFNt5FDizXXZU81tgyU6280ua\nYDNSze3AK3dSs4rmUnNJkjRN+ewnTRuLF/femFoTzTGffI755HPM67FbdxTeWyWZD6xZs2aNJ5dJ\nkjQGa9euZcGCBdA8NWDteG7bmRpJklQFQ40kSaqCoUaSJFXBUCNJkqqwO89+kjTJNm7cyMDAwM4L\nx8msWbOYM2fOpO1PknaHoUaaJjZu3MjcufPYtu3hSdvnQQc9jfXr1xlsJE0LhhppmhgYGGgDzZXA\nZDxIdR3bti1hYGDAUCNpWjDUSNPOPMD7I0lSL08UliRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJU\nBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaS\nJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCo\nkSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQq\nGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUYU6hJ8tdJVifZkmRzkm8k\n+aNh6s5Lcl+Sh5N8N8kRPe0HJrk0yUCSh5Jck+RZPTXPSPK1JINJHkxyWZKDe2oOS3Jdkq1JNiW5\nMMk+PTVHJVmV5JEk9yQ5e5j+nphkTZJtSe5KcsZYxkWSJE29sc7UvAL4DHAc8KfA/sD1SX5vqCDJ\nh4H3A+8CjgW2AiuTHNDZzsXAqcDrgIXAc4Cv9+zrKmAesKitXQh8obOffYAVwH7A8cAZwFuB8zo1\nTwdWAhuA+cDZwLIk7+zUHA5cC9wIvAT4NHBZklePaWQkSdKU2m8sxaWU13RfJ3kr8BtgAfDDdvVZ\nwPmllGvbmrcAm4G/BK5OMgN4O/CmUsoP2pq3AeuSHFtKWZ1kHnAysKCUcmtbcyZwXZIPlVI2te0v\nBF5VShkAbkvyMeATSZaVUp4AltAEr3e0r9clORr4IHBZ29/3AL8opZzTvl6f5OXAUuC7YxkfSZI0\ndXb3nJpDgAI8AJDk+cBsmlkPAEopW4BbgBPaVS+lCVPdmvXAxk7N8cCDQ4GmdUO7r+M6Nbe1gWbI\nSmAm8OJOzao20HRr5iaZ2am5oed9rez0RZIkTQO7HGqShOYw0g9LKXe0q2fTBI/NPeWb2zaAPuCx\nNuzsqGY2zQzQ75RSnqQJT92a4fbDONXMSHIgkiRpWhjT4acenwVeBLxsnPoymTIeG1m6dCkzZ858\nyrrFixezePHi8di8JEnTWn9/P/39/U9ZNzg4OGH726VQk+QS4DXAK0opv+40baIJDH08dfajD7i1\nU3NAkhk9szV9bdtQTe/VUPsCh/bUHNPTtb5O29DXvmFqyihqtpRSHmUEy5cvZ/78+SOVSJK01xru\nH/pr165lwYIFE7K/MR9+agPNX9CcoLux21ZK2UATEhZ16mfQnAfzo3bVGuCJnpq5wBzg5nbVzcAh\n7Um9QxbRBKZbOjVHJpnVqTkJGATu6NQsbANRt2Z9KWWwU7OIpzqp0xdJkjQNjPU+NZ8F3gycDmxN\n0tcuB3XKLgY+muTPkxwJfAW4F/gm/O7E4cuBi9r7wywArgBuKqWsbmvupDlZ94tJjknyMppLyfvb\nK58ArqcJL19t70VzMnA+cEkp5fG25irgMeCKJC9K8kbgA8CnOv39PPCCJBckmZvkvcDrgYvGMjaS\nJGlqjfXw07tpDt18v2f922jCC6WUC5M8jeaeMocA/wicUkp5rFO/FHgSuAY4EPgO8L6ebZ4OXEJz\nZdL2tvasocZSyvYkpwGfo5kF2gp8GTi3U7MlyUnApcBPgAFgWSnl8k7N3UlOBZbTBJ57aS4B770i\nSpIk7cHGep+aUc3slFKWActGaH8UOLNddlTzW5r7zIy0n18Cp+2k5nbglTupWUVzrx1JkjRN+ewn\nSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqC\noUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mS\nqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRI\nkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUM\nNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJU\nBUONJEmqgqFGkiRVYcyhJskrknwrya+SbE/y2p72L7Xru8uKnpoDk1yaZCDJQ0muSfKsnppnJPla\nksEkDya5LMnBPTWHJbkuydYkm5JcmGSfnpqjkqxK8kiSe5KcPcx7OjHJmiTbktyV5IyxjoskSZpa\nuzJTczDwU+C9QNlBzbeBPmB2uyzuab8YOBV4HbAQeA7w9Z6aq4B5wKK2diHwhaHGNrysAPYDjgfO\nAN4KnNepeTqwEtgAzAfOBpYleWen5nDgWuBG4CXAp4HLkrx6xFGQJEl7lP3G+g2llO8A3wFIkh2U\nPVpKuX+4hiQzgLcDbyql/KBd9zZgXZJjSymrk8wDTgYWlFJubWvOBK5L8qFSyqa2/YXAq0opA8Bt\nST4GfCLJslLKE8ASYH/gHe3rdUmOBj4IXNZ26T3AL0op57Sv1yd5ObAU+O5Yx0eSJE2NiTqn5sQk\nm5PcmeSzSQ7ttC2gCVM3Dq0opawHNgIntKuOBx4cCjStG2hmho7r1NzWBpohK4GZwIs7NavaQNOt\nmZtkZqfmhp7+r+z0RZIkTQMTEWq+DbwF+BPgHOCVwIrOrM5s4LFSypae79vctg3V/KbbWEp5Enig\np2bzMNtgnGpmJDlwmPcnSZL2QGM+/LQzpZSrOy9/luQ24OfAicD3xnt/u2hHh83GZOnSpcycOfMp\n6xYvXszixb2nEEmStPfp7++nv7//KesGBwcnbH/jHmp6lVI2JBkAjqAJNZuAA5LM6Jmt6WvbaL/2\nXg21L3BoT80xPbvr67QNfe0bpqaMomZLKeXRkd7b8uXLmT9//kglkiTttYb7h/7atWtZsGDBhOxv\nwu9Tk+S5wDOBX7er1gBP0FzVNFQzF5gD3Nyuuhk4pD2pd8gimhmWWzo1RyaZ1ak5CRgE7ujULGwD\nUbdmfSllsFOziKc6qdMXSZI0DezKfWoOTvKSJP+xXfWC9vVhbduFSY5L8rwki4B/AO6iOfmWdnbm\ncuCi9v4wC4ArgJtKKavbmjvb+i8mOSbJy4DPAP3tlU8A19OEl6+296I5GTgfuKSU8nhbcxXwGHBF\nkhcleSPwAeBTnbf0+fY9XJBkbpL3Aq8HLhrr2EiSpKmzK4efXkpzGKm0y1BA+Duae9ccRXOi8CHA\nfTTh5G86QQOay6WfBK4BDqS5RPx9Pfs5HbiE5sqk7W3tWUONpZTtSU4DPgf8CNgKfBk4t1OzJclJ\nwKXAT4ABYFkp5fJOzd1JTgWW0wSee2kuAe+9IkqSJO3BduU+NT9g5BmePxvFNh4FzmyXHdX8luY+\nMyNt55fAaTupuZ3mCqyRalbRXGouSZKmKZ/9JEmSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYa\nSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqC\noUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mS\nqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRI\nkqQqGGokSVIVDDWSJKkK+011ByTt2datWzdp+5o1axZz5syZtP1JqouhRtIO/BrYhyVLlkzaHg86\n6GmsX7/OYCNplxhqJO3Ab4HtwJXAvEnY3zq2bVvCwMCAoUbSLjHUSNqJecD8qe6EJO2UJwpLkqQq\nGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhXGHGqS\nvCLJt5L8Ksn2JK8dpua8JPcleTjJd5Mc0dN+YJJLkwwkeSjJNUme1VPzjCRfSzKY5MEklyU5uKfm\nsCTXJdmaZFOSC5Ps01NzVJJVSR5Jck+Ss4fp74lJ1iTZluSuJGeMdVwkSdLU2pWZmoOBnwLvBUpv\nY5IPA+8H3gUcC2wFViY5oFN2MXAq8DpgIfAc4Os9m7qK5qEzi9rahcAXOvvZB1hB8/yq44EzgLcC\n53Vqng6sBDbQPLzmbGBZknd2ag4HrgVuBF4CfBq4LMmrRzkekiRpDzDmB1qWUr4DfAcgSYYpOQs4\nv5RybVvzFmAz8JfA1UlmAG8H3lRK+UFb8zZgXZJjSymrk8wDTgYWlFJubWvOBK5L8qFSyqa2/YXA\nq0opA8BtST4GfCLJslLKE8ASYH/gHe3rdUmOBj4IXNb29z3AL0op57Sv1yd5ObAU+O5Yx0eSJE2N\ncT2nJsnzgdk0sx4AlFK2ALcAJ7SrXkoTpro164GNnZrjgQeHAk3rBpqZoeM6Nbe1gWbISmAm8OJO\nzao20HRr5iaZ2am5oeetrOz0RZIkTQPjfaLwbJrgsbln/ea2DaAPeKwNOzuqmQ38pttYSnkSeKCn\nZrj9ME41M5IciCRJmhbGfPipEsMdNhuzpUuXMnPmzKesW7x4MYsXLx6PzUuSNK319/fT39//lHWD\ng4MTtr/xDjWbaAJDH0+d/egDbu3UHJBkRs9sTV/bNlTTezXUvsChPTXH9Oy/r9M29LVvmJoyipot\npZRH/+1b/FfLly9n/vz5I5Wochs3bmRgYGDnheNg3bp1k7IfSRovw/1Df+3atSxYsGBC9jeuoaaU\nsiHJJporlv4PQHti8HHApW3ZGuCJtuYbbc1cYA5wc1tzM3BIkqM759UsoglMt3RqPpJkVue8mpOA\nQeCOTs1/S7Jve/hqqGZ9KWWwU3NKz1s5qdMXaVgbN25k7tx5bNv28FR3RZLELoSa9l4xR/Cvh3Be\nkOQlwAOllF/SXK790ST/DNwNnA/cC3wTmhOHk1wOXJTkQeAh4G+Bm0opq9uaO5OsBL6Y5D3AAcBn\ngP72yieA62nCy1fby8if3e7rklLK423NVcDfAFckuQA4EvgAzRVaQz4PvK9tv4ImPL0eeM1Yx0Z7\nl4GBgTbQXElz94GJtgL42CTsR5Kmp12ZqXkp8D2aQzgF+FS7/u+At5dSLkzyNJp7yhwC/CNwSinl\nsc42lgJPAtcAB9JcIv6+nv2cDlxCc2XS9rb2d2GklLI9yWnA54Af0dwP58vAuZ2aLUlOopkl+gkw\nACwrpVzeqbk7yanAcprAcy/NJeC9V0RJOzCP5jZIE83DT5I0kl25T80P2MlVU6WUZcCyEdofBc5s\nlx3V/JbmPjMj7eeXwGk7qbkdeOVOalYBE3OAT5IkTQqf/SRJkqpgqJEkSVUw1EiSpCoYaiRJUhUM\nNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJU\nBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaS\nJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCo\nkSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQq\nGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCqMe6hJcm6S7T3L\nHT015yW5L8nDSb6b5Iie9gOTXJpkIMlDSa5J8qyemmck+VqSwSQPJrksycE9NYcluS7J1iSbklyY\nZJ+emqOSrErySJJ7kpw93mMiSZIm3kTN1NwO9AGz2+XlQw1JPgy8H3gXcCywFViZ5IDO918MnAq8\nDlgIPAf4es8+rgLmAYva2oXAFzr72QdYAewHHA+cAbwVOK9T83RgJbABmA+cDSxL8s7deO+SJGkK\n7DdB232ilHL/DtrOAs4vpVwLkOQtwGbgL4Grk8wA3g68qZTyg7bmbcC6JMeWUlYnmQecDCwopdza\n1pwJXJfkQ6WUTW37C4FXlVIGgNuSfAz4RJJlpZQngCXA/sA72tfrkhwNfBC4bPyHRZIkTZSJmqn5\nwyS/SvLzJFcmOQwgyfNpZm5uHCospWwBbgFOaFe9lCZsdWvWAxs7NccDDw4FmtYNQAGO69Tc1gaa\nISuBmcCLOzWr2kDTrZmbZOYuvXNJkjQlJiLU/JjmMM/JwLuB5wOr2vNdZtMEj80937O5bYPmsNVj\nbdjZUc1s4DfdxlLKk8ADPTXD7Ycx1kiSpGlg3A8/lVJWdl7enmQ1cA/wBuDO8d7fVFq6dCkzZz51\nQmfx4sUsXrx4inokSdKeo7+/n/7+/qesGxwcnLD9TdQ5Nb9TShlMchdwBPB9IDSzMd0Zkj5g6FDS\nJuCAJDN6Zmv62rahmt6rofYFDu2pOaanO32dtqGvfTup2aHly5czf/78nZVJkrRXGu4f+mvXrmXB\nggUTsr8Jv09Nkt+nCTT3lVI20ISFRZ32GTTnwfyoXbUGeKKnZi4wB7i5XXUzcEh7Uu+QRTSB6ZZO\nzZFJZnVqTgIGgTs6NQvbQNStWV9KmbgoKUmSxt1E3Kfmk0kWJnlekj8GvgE8Dvx9W3Ix8NEkf57k\nSOArwL3AN+F3Jw5fDlyU5MQkC4ArgJtKKavbmjtpTuj9YpJjkrwM+AzQ3175BHA9TXj5ansvmpOB\n84FLSimPtzVXAY8BVyR5UZI3Ah8APjXe4yJJkibWRBx+ei5NWHgmcD/wQ+D4Usq/AJRSLkzyNJp7\nyhwC/CNwSinlsc42lgJPAtcABwLfAd7Xs5/TgUtornra3taeNdRYStme5DTgczSzQFuBLwPndmq2\nJDkJuBT4CTAALCulXL7boyBJkibVRJwovNOzZEspy4BlI7Q/CpzZLjuq+S3NfWZG2s8vgdN2UnM7\n8MqRaiRJ0p7PZz9JkqQqGGokSVIVDDWSJKkKhhpJklSFCb/5niSNxbp16yZlP7NmzWLOnDmTsi9J\nk8NQI2kP8WtgH5YsGfGixnFz0EFPY/36dQYbqSKGGkl7iN/S3HLqSmDeBO9rHdu2LWFgYMBQI1XE\nUCNpDzMP8JlqksbOE4UlSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVXwPjWS\n9lqT9UiGqeBjILQ3MtRI2gtN7iMZpoKPgdDeyFAjaS80mY9kmAo+BkJ7J0ONpL2Yj2SQauKJwpIk\nqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBR+T\nIEmVqvkp5LXzKeu7xlAjSdWp/ynktfMp67vGUCNJ1an9KeS18ynru8pQI0nV8ink2rt4orAkSaqC\noUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoJ3FNaE2rhxIwMD\nA1PdjQnhwwIlac9iqNGE2bhxI3PnzmPbtoenuiuSpL2AoUYTZmBgoA00tT5UbwXwsanuhCSpZajR\nJKj1oXoefpKkPYknCkuSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjaaR/qnugDQJ/D2ffI55LQw1rSTv\nS7IhySNJfpzkmKnuk3r5waO9gb/nk88xr4WhBkjyRuBTwLnA0cA/ASuTzJrSjkmSpFEz1DSWAl8o\npXyllHIn8G7gYeDtU9stSZI0Wnt9qEmyP7AAuHFoXSmlADcAJ0xVvyRJ0th4R2GYBewLbO5ZvxmY\nu4PvOQgm9oGG999//7R/EOSGDRva/1rB+Nx9917ga+OwnfFyU/t1vN6f+5u8/U32exuL8fg935Pf\n355oT/tsaT47a31obud9HTTe204zKbH3SvJs4FfACaWUWzrrLwAWllL+zWxNktPZs/4PkCRpunlz\nKeWq8dygMzUwADwJ9PWs7wM27eB7VgJvBu4Gtk1YzyRJqs9BwOE0f0vH1V4/UwOQ5MfALaWUs9rX\nATYCf1tK+eSUdk6SJI2KMzWNi4AvJ1kDrKa5GuppwJenslOSJGn0DDVAKeXq9p4059EcdvopcHIp\n5f6p7ZkkSRotDz9JkqQq7PX3qZEkSXUw1EiSpCoYakaQ5CNJbkqyNckDO6g5LMl1bc2mJBcm2aen\n5qgkq9qHZd6T5OzJeQfTX5K7k2zvLE8mOaenZqc/A42ND3idOEnO7fmd3p7kjp6a85Lcl+ThJN9N\ncsRU9Xc6SvKKJN9K8qt2fF87TM2IY5zkwCSXJhlI8lCSa5I8a/LexfSyszFP8qVhfu9X9NTs9pj7\nwT+y/YGrgc8N19j+4VxBc8L18cAZwFtpTjgeqnk6zbX4G4D5wNnAsiTvnMiOV6QAH6U5gXs28Gzg\nM0ONo/kZaGx8wOukuJ1//Z2eDbx8qCHJh4H3A+8CjgW20oz/AVPQz+nqYJoLPt5L8xnyFKMc44uB\nU4HXAQuB5wBfn9huT2sjjnnr2zz1935xT/vuj3kpxWUnC80fygeGWX8K8Dgwq7Pur4AHgf3a1++h\nucHffp2ajwN3TPX7mg4LTRj8wAjtO/0ZuIx5zH8MfLrzOjT3kT9nqvtWw0ITFteO0H4fsLTzegbw\nCPCGqe77dFyA7cBrxzLG7etHgf/UqZnbbuvYqX5Pe/qygzH/EvA/R/iecRlzZ2p2z/HAbaWU7kOa\nVgIzgRd3alaVUp7oqZmbZObkdHPa+y/tdOTaJB9Ksm+nbTQ/A42SD3idNH/YTtP/PMmVSQ4DSPJ8\nmn/Bdsd/C3ALjv+4GOUYv5Rm9rdbs57mpqz+HHbdiUk2J7kzyWeTHNppW8A4jLmhZvfMZvgHYQ61\njbZGO/Zp4E3AicDngY8AF3TaHd/xNdIDXh3P8fFjmkOkJwPvBp4PrEpyMM0YFxz/iTSaMe4DHmvD\nzo5qNDbzuuCbAAADCUlEQVTfBt4C/AlwDvBKYEV7B39oxnW3x3yvu/leko8DHx6hpADzSil3TVKX\n9jpj+RmUUi7urL89yWPAF5L8dSnl8QntqDQBSind593cnmQ1cA/wBuDOqemVNLFKKVd3Xv4syW3A\nz2n+wfq98drPXhdqgP9Oc2xvJL8Y5bY2Ab1XhfR12oa+DvewzG7N3mZ3fgaraX5vDwf+L6P7GWj0\nduUBr9oNpZTBJHcBRwDfpzmHqY+nziT0AbdOfu+qtImdj/Em4IAkM3pmDvz/YJyUUjYkGaD5vf8e\n4zTme93hp1LKv7QzACMtT+x8SwDcDBzZc1XIScAgcEenZmHPeSAnAetLKYO7/Yamod38GRxNc+LY\nb9rXo/kZaJTa2a81wKKhde308CLgR1PVr5ol+X2aD/b7SikbaD7Au+M/AzgOx39cjHKM1wBP9NTM\nBebQfOZoNyV5LvBM4NftqnEZ871xpmbU2pP3DgWeB+yb5CVt0z+XUrYC19P84fxqe4ngs4HzgUs6\nh0auAv4GuCLJBcCRwAeAsybvnUxPSY6n+aD5HvAQ8Mc0Dx/9aicQjuZnoLHxAa8TKMkngf9Fc8jp\n3wH/leYKvr9vSy4GPprkn4G7aX6f7wW+Oemdnaba85OOoJmRAXhB+/n9QCnll+xkjEspW5JcDlyU\n5EGaz5+/BW4qpaye1DczTYw05u1yLs3l2ZvauguAu2gu7Bi/MZ/qS7/25IXmEMmTwywLOzWHAdcC\n/49mKvMCYJ+e7fwH4AfAwzRncn9oqt/bdFhoZmVubv+H2Epzb49zgP176nb6M3AZ89i/l+bD/pH2\nZ/DSqe5TLQvQT/MH9JH28+Aq4Pk9NctoLjt+uP3QP2Kq+z2dFpqTULcP89l9xWjHGDiQ5p5YA+0f\n2P8BPGuq39ueuow05sBBwHdoAs02mtMLPgf8wXiPuQ+0lCRJVdjrzqmRJEl1MtRIkqQqGGokSVIV\nDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhX+P1lUlZOgleMg\nAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10d155780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.mlab as mlab\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Look at overall histogram\n",
    "data_tuple = ml_bucketized_features\\\n",
    "  .select(\"ArrDelay\")\\\n",
    "  .rdd\\\n",
    "  .flatMap(lambda x: x)\\\n",
    "  .histogram([-87.0, -60, -30, -15, 0, 15, 30, 60, 120])\n",
    "\n",
    "heights = np.array(data_tuple[1])\n",
    "\n",
    "# The bins are 1 > length than the values\n",
    "full_bins = data_tuple[0]\n",
    "\n",
    "# Bars are drawn from the left\n",
    "mid_point_bins = full_bins[:-1]\n",
    "\n",
    "# The width of a bar should be the range it maps in the data\n",
    "widths = [abs(i - j) for i, j in zip(full_bins[:-1], full_bins[1:])]\n",
    "\n",
    "# And now the bar should plot nicely\n",
    "bar = plt.bar(mid_point_bins, heights, width=widths, color='b')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see here that there is a strong mean in our data around 0-15, with some right left, and that most of the data was falling either in `ArrDelayBucket` `0.0` or very nearly so in the next bin. This must make the bins hard to reckon for the classifier."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Repartitioning `ArrDelayBucket`\n",
    "This histogram seems to indicate that we might address our problem by re-bucketizing `ArrDelayBucket` into something more balanced. While the original schema of `<15,15-60,60+` seemed to make sense from a user perspective... it seems to have wrecked our model. So lets try something that respects the distribution of delay...\n",
    "\n",
    "### Using Histograms to Determine Balanced Buckets\n",
    "Before we dive in and change our buckets, lets play with defining different buckets for our histogram and see if we can create balanced bars! To play around with this without repeating ourselves, lets create a function `create_hist` to help."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_hist(rdd_histogram_data):\n",
    "  \"\"\"Given an RDD.histogram, plot a pyplot histogram\"\"\"\n",
    "  heights = np.array(rdd_histogram_data[1])\n",
    "  full_bins = rdd_histogram_data[0]\n",
    "  mid_point_bins = full_bins[:-1]\n",
    "  widths = [abs(i - j) for i, j in zip(full_bins[:-1], full_bins[1:])]\n",
    "  bar = plt.bar(mid_point_bins, heights, width=widths, color='b')\n",
    "  return bar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing the Original Bucketing Schema\n",
    "First, lets check out the distribution of the data using the original bucketing scheme. Although the maximum arrival delay is 1971 minutes, let use 200 to keep our chart from distorting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Container object of 3 artists>"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAFkCAYAAADYL8pXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3X2wXWVh7/HvLwRCoSaolBMdQXBoY7RASXizStSmhotg\na0enEmXEF8aKitw4ArWjJQN3rgWvBCuojEB9Qc4dLt5ee4EawbZGAWFIqIUSwr3lTcTEHoETbngJ\nkOf+sdbRld2Tlx1OspPnfD8zezJ7rV/2evYzOzm/s952SilIkiTt6qYMegCSJEkTwVIjSZKqYKmR\nJElVsNRIkqQqWGokSVIVLDWSJKkKlhpJklQFS40kSaqCpUaSJFXBUiNJkqrQV6lJck6SDT2Pu3sy\n5yZ5JMmTSW5IcnDP+mlJLkkykuSJJNck2a8n8+Ik30oymuSxJJcl2bsns3+S65KsS7I6yQVJpvRk\nDk2yLMlTSR5McuY47+lNSZYneTrJvUlO6WdOJEnSzmFb9tTcBQwBM9vHG8ZWJDkb+BjwIeAoYB2w\nNMkenb9/EXAC8A5gHvBy4Ns927gKmA3Mb7PzgEs725kCXA9MBY4BTgHeB5zbybwIWArcD8wBzgQW\nJzm1kzkQuBb4PnAY8AXgsiRv6W9KJEnSoKWfL7RMcg7wx6WUOZtY/wjwuVLKkvb5dGANcEop5er2\n+b8DJ5VS/rbNzAJWAseUUm5LMhv4V2BuKeWONnMccB3wilLK6iTHA38HvKyUMtJm/gz4K+C3SinP\nJTkNOA+YWUp5rs18th3/a9rn5wPHl1IO7byHYWBGKeWtWz0xkiRp4LZlT81vJ/lZkn9LcmWS/QGS\nHESz5+b7Y8FSylrgVuB17aIjaPaudDOrgIc6mWOAx8YKTetGoABHdzJ3jhWa1lJgBvDaTmbZWKHp\nZGYlmdHJ3Njz/pZ2xiJJknYRU/vM/5jmMM8q4GXAYmBZkt+lKTSFZs9M15p2HTSHrda3ZWdTmZnA\nL7orSynPJ3m0JzPedsbW/aT9877NZEY38zrTk0wrpTzDOJK8FDgOeAB4eryMJEka157AgcDSUsov\nJ/KF+yo1pZSlnad3JbkNeBD4U+CeiRzYdpQJeI3jgG9NwOtIkjRZvYfmHNoJ0++emo2UUkaT3Asc\nDPwTTWEYYuO9H0PA2KGk1cAeSab37K0ZateNZXqvhtoNeElP5sie4Qx11o39OTROpmxFZu2m9tK0\nHgC48sormT179mZi6lq0aBFLliwZ9DB2Oc5b/5yzbeO89c8569/KlSs5+eSTof1ZOpFeUKlJ8ps0\nhebrpZT7k6ymuWLpX9r102nOg7mk/SvLgefaTPdE4QOAW9rMLcA+SQ7vnFczn6Yw3drJ/EWSfTvn\n1SygOaR0dyfzX5LsVkp5vpNZVUoZ7WSO73lbCzpj2ZSnAWbPns2cOeOeM61xzJgxw/naBs5b/5yz\nbeO89c85e0Em/PSNfu9T87kk85K8Msnv0xSTZ4H/3kYuAj6d5G1JDgG+ATwMfAd+deLw5cCF7f1h\n5gJXADeVUm5rM/fQnKz71SRHJnk98EVguJQytoflezTl5ZvtvWiOo7nS6eJSyrNt5ipgPXBFktck\neRfwceDznbf0FeBVSc5PMivJR4B3Ahf2My+SJGnw+t1T8wqasvBSmkuzf0RzKfYvAUopFyTZi+ae\nMvsAP6S5ZHp95zUWAc8D1wDTgO8CH+3ZzruBi2muTNrQZs8YW1lK2ZDkRODLwM0098P5GnBOJ7M2\nyQKavUS3AyPA4lLK5Z3MA0lOAJbQFJ6HgQ+WUnqviJIkSTu5fk8UXrgVmcU0V0Vtav0zwOntY1OZ\nx4GTt7CdnwInbiFzF/DGLWSWAXM3l5EkSTs/v/tJO8zChVvsxBqH89Y/52zbOG/9c852Ln3dUViN\nJHOA5cuXL/cEMUmS+rBixQrmzp0LzTcHrJjI13ZPjSRJqoKlRpIkVcFSI0mSqmCpkSRJVbDUSJKk\nKlhqJElSFSw1kiSpCpYaSZJUBUuNJEmqgqVGkiRVwVIjSZKqYKmRJElVsNRIkqQqTB30ANR46KGH\nGBkZGfQwNMnsu+++HHDAAYMehiRNCEvNTuChhx5i1qzZPP30k4MeiiaZPffci1WrVlpsJFXBUrMT\nGBkZaQvNlcDsQQ9Hk8ZKnn76ZEZGRiw1kqpgqdmpzAbmDHoQkiTtkjxRWJIkVcFSI0mSqmCpkSRJ\nVbDUSJKkKlhqJElSFSw1kiSpCpYaSZJUBUuNJEmqgqVGkiRVwVIjSZKqYKmRJElVsNRIkqQqWGok\nSVIVLDWSJKkKlhpJklQFS40kSaqCpUaSJFXBUiNJkqpgqZEkSVWw1EiSpCpYaiRJUhUsNZIkqQqW\nGkmSVAVLjSRJqoKlRpIkVcFSI0mSqmCpkSRJVbDUSJKkKlhqJElSFSw1kiSpCpYaSZJUBUuNJEmq\nwgsqNUn+PMmGJBf2LD83ySNJnkxyQ5KDe9ZPS3JJkpEkTyS5Jsl+PZkXJ/lWktEkjyW5LMnePZn9\nk1yXZF2S1UkuSDKlJ3NokmVJnkryYJIzx3kfb0qyPMnTSe5NcsoLmRdJkrTjbXOpSXIk8CHgJz3L\nzwY+1q47ClgHLE2yRyd2EXAC8A5gHvBy4Ns9m7gKmA3Mb7PzgEs725kCXA9MBY4BTgHeB5zbybwI\nWArcD8wBzgQWJzm1kzkQuBb4PnAY8AXgsiRv6Wc+JEnSYG1TqUnym8CVwKnA4z2rzwDOK6VcW0q5\nC3gvTWl5e/t3pwMfABaVUn5QSrkDeD/w+iRHtZnZwHHAB0spt5dSbgZOB05KMrPdznHAq4H3lFLu\nLKUsBT4DfDTJ1DZzMrB7+zorSylXA38NfKIz3tOA+0opZ5VSVpVSLgGuARZty9xIkqTB2NY9NZcA\n/7uU8g/dhUkOAmbS7PUAoJSyFrgVeF276AiavSvdzCrgoU7mGOCxtvCMuREowNGdzJ2llJFOZikw\nA3htJ7OslPJcT2ZWkhmdzI09729pZyySJGkX0HepSXIS8HvAp8ZZPZOmeKzpWb6mXQcwBKxvy86m\nMjOBX3RXllKeBx7tyYy3HSYoMz3JNCRJ0i5h6pYjv5bkFTTnw/xhKeXZ7TOkHSIT8SKLFi1ixowZ\nGy1buHAhCxcunIiXlyRplzY8PMzw8PBGy0ZHR7fb9voqNcBc4LeAFUnGisFuwLwkH6M5xyU0e2O6\nez+GgLFDSauBPZJM79lbM9SuG8v0Xg21G/CSnsyRPeMb6qwb+3NonEzZiszaUsozbMaSJUuYM2fO\n5iKSJE1a4/2iv2LFCubOnbtdttfv4acbgUNoDj8d1j5upzlp+LBSyn00JWH+2F9oTww+Gri5XbQc\neK4nMws4ALilXXQLsE+Swzvbnk9TmG7tZA5Jsm8nswAYBe7uZOa1haibWVVKGe1k5rOxBZ2xSJKk\nXUBfe2pKKev4dWEAIMk64JellJXtoouATyf5v8ADwHnAw8B32tdYm+Ry4MIkjwFP0FyRdFMp5bY2\nc0+SpcBXk5wG7AF8ERgupYztYfleO5ZvtpeRv6zd1sWdQ2NXAX8JXJHkfJpC9nGaK7TGfIXmiqnz\ngStoCs47gbf2MzeSJGmw+j38NJ6y0ZNSLkiyF809ZfYBfggcX0pZ34ktAp6nuXR6GvBd4KM9r/tu\n4GKavUMb2uyvykgpZUOSE4Ev0+wFWgd8DTink1mbZAHN1Vq3AyPA4lLK5Z3MA0lOAJbQFJ6HaS4B\n770iSpIk7cRecKkppfzBOMsWA4s383eeobnvzOmbyTxOc5+ZzW37p8CJW8jcBbxxC5llNOcLSZKk\nXZTf/SRJkqpgqZEkSVWw1EiSpCpYaiRJUhUsNZIkqQqWGkmSVAVLjSRJqoKlRpIkVcFSI0mSqmCp\nkSRJVbDUSJKkKlhqJElSFSw1kiSpCpYaSZJUBUuNJEmqgqVGkiRVwVIjSZKqYKmRJElVsNRIkqQq\nWGokSVIVLDWSJKkKlhpJklQFS40kSaqCpUaSJFXBUiNJkqpgqZEkSVWw1EiSpCpYaiRJUhUsNZIk\nqQqWGkmSVAVLjSRJqoKlRpIkVcFSI0mSqmCpkSRJVbDUSJKkKlhqJElSFSw1kiSpCpYaSZJUBUuN\nJEmqgqVGkiRVwVIjSZKqYKmRJElVsNRIkqQqWGokSVIVLDWSJKkKlhpJklQFS40kSaqCpUaSJFXB\nUiNJkqpgqZEkSVWw1EiSpCr0VWqSfDjJT5KMto+bk/ynnsy5SR5J8mSSG5Ic3LN+WpJLkowkeSLJ\nNUn268m8OMm32m08luSyJHv3ZPZPcl2SdUlWJ7kgyZSezKFJliV5KsmDSc4c5z29KcnyJE8nuTfJ\nKf3MiSRJ2jn0u6fmp8DZwBxgLvAPwHeSzAZIcjbwMeBDwFHAOmBpkj06r3ERcALwDmAe8HLg2z3b\nuQqYDcxvs/OAS8dWtuXlemAqcAxwCvA+4NxO5kXAUuD+drxnAouTnNrJHAhcC3wfOAz4AnBZkrf0\nOS+SJGnApvYTLqVc17Po00lOoykWK4EzgPNKKdcCJHkvsAZ4O3B1kunAB4CTSik/aDPvB1YmOaqU\ncltbkI4D5pZS7mgzpwPXJflkKWV1u/7VwJtLKSPAnUk+A/xVksWllOeAk4HdgQ+2z1cmORz4BHBZ\nO/7TgPtKKWe1z1cleQOwCLihn7mRJEmDtc3n1CSZkuQkYC/g5iQHATNp9noAUEpZC9wKvK5ddARN\nkepmVgEPdTLHAI+NFZrWjUABju5k7mwLzZilwAzgtZ3MsrbQdDOzkszoZG7seWtLO2ORJEm7iL5L\nTZLfTfIE8AzwJeBP2mIyk6Z4rOn5K2vadQBDwPq27GwqMxP4RXdlKeV54NGezHjbYYIy05NMQ5Ik\n7TL6OvzUuofm/JMZwDuBbySZN6Gj2r4yUS+0aNEiZsyYsdGyhQsXsnDhwonahCRJu6zh4WGGh4c3\nWjY6Orrdttd3qWkP59zXPr0jyVE059JcQFMYhth478cQMHYoaTWwR5LpPXtrhtp1Y5neq6F2A17S\nkzmyZ2hDnXVjfw6NkylbkVlbSnmGLViyZAlz5szZUkySpElpvF/0V6xYwdy5c7fL9ibiPjVTgGml\nlPtpSsL8sRXticFHAze3i5YDz/VkZgEHALe0i24B9mlP6h0zn6Yw3drJHJJk305mATAK3N3JzGsL\nUTezqpQy2snMZ2MLOmORJEm7iH7vU/Nfkxyb5JXtuTWfBd4IXNlGLqK5IuptSQ4BvgE8DHwHfnXi\n8OXAhe39YeYCVwA3lVJuazP30Jys+9UkRyZ5PfBFYLi98gngezTl5ZvtvWiOA84DLi6lPNtmrgLW\nA1ckeU2SdwEfBz7feUtfAV6V5Pwks5J8hOaQ2oX9zIskSRq8fg8/7Qd8HXgZzV6RfwEWlFL+AaCU\nckGSvWjuKbMP8EPg+FLK+s5rLAKeB64BpgHfBT7as513AxfTXJm0oc2eMbaylLIhyYnAl2n2Aq0D\nvgac08msTbIAuAS4HRgBFpdSLu9kHkhyArCEpvA8THMJeO8VUZIkaSfX731qTt2KzGJg8WbWPwOc\n3j42lXmc5j4zm9vOT4ETt5C5i2ZP0uYyy2huJChJknZhfveTJEmqgqVGkiRVwVIjSZKqYKmRJElV\nsNRIkqQqWGokSVIVLDWSJKkKlhpJklQFS40kSaqCpUaSJFXBUiNJkqpgqZEkSVWw1EiSpCpYaiRJ\nUhUsNZIkqQqWGkmSVAVLjSRJqoKlRpIkVcFSI0mSqmCpkSRJVbDUSJKkKlhqJElSFSw1kiSpCpYa\nSZJUBUuNJEmqgqVGkiRVwVIjSZKqYKmRJElVsNRIkqQqWGokSVIVLDWSJKkKlhpJklQFS40kSaqC\npUaSJFXBUiNJkqpgqZEkSVWw1EiSpCpYaiRJUhUsNZIkqQqWGkmSVAVLjSRJqoKlRpIkVcFSI0mS\nqmCpkSRJVbDUSJKkKlhqJElSFSw1kiSpCpYaSZJUBUuNJEmqgqVGkiRVwVIjSZKq0FepSfKpJLcl\nWZtkTZK/TfI74+TOTfJIkieT3JDk4J7105JckmQkyRNJrkmyX0/mxUm+lWQ0yWNJLkuyd09m/yTX\nJVmXZHWSC5JM6ckcmmRZkqeSPJjkzHHG+6Yky5M8neTeJKf0My+SJGnw+t1TcyzwReBo4A+B3YHv\nJfmNsUCSs4GPAR8CjgLWAUuT7NF5nYuAE4B3APOAlwPf7tnWVcBsYH6bnQdc2tnOFOB6YCpwDHAK\n8D7g3E7mRcBS4H5gDnAmsDjJqZ3MgcC1wPeBw4AvAJcleUtfMyNJkgZqaj/hUspbu8+TvA/4BTAX\n+FG7+AzgvFLKtW3mvcAa4O3A1UmmAx8ATiql/KDNvB9YmeSoUsptSWYDxwFzSyl3tJnTgeuSfLKU\nsrpd/2rgzaWUEeDOJJ8B/irJ4lLKc8DJNMXrg+3zlUkOBz4BXNaO9zTgvlLKWe3zVUneACwCbuhn\nfiRJ0uC80HNq9gEK8ChAkoOAmTR7PQAopawFbgVe1y46gqZMdTOrgIc6mWOAx8YKTevGdltHdzJ3\ntoVmzFJgBvDaTmZZW2i6mVlJZnQyN/a8r6WdsUiSpF3ANpeaJKE5jPSjUsrd7eKZNMVjTU98TbsO\nYAhY35adTWVm0uwB+pVSyvM05ambGW87TFBmepJpSJKkXUJfh596fAl4DfD6CRrLjpSJeJFFixYx\nY8aMjZYtXLiQhQsXTsTLS5K0SxseHmZ4eHijZaOjo9tte9tUapJcDLwVOLaU8vPOqtU0hWGIjfd+\nDAF3dDJ7JJnes7dmqF03lum9Gmo34CU9mSN7hjbUWTf259A4mbIVmbWllGfYjCVLljBnzpzNRSRJ\nmrTG+0V/xYoVzJ07d7tsr+/DT22h+WOaE3Qf6q4rpdxPUxLmd/LTac6DubldtBx4riczCzgAuKVd\ndAuwT3tS75j5NIXp1k7mkCT7djILgFHg7k5mXluIuplVpZTRTmY+G1vQGYskSdoF9Hufmi8B7wHe\nDaxLMtQ+9uzELgI+neRtSQ4BvgE8DHwHfnXi8OXAhe39YeYCVwA3lVJuazP30Jys+9UkRyZ5Pc2l\n5MPtlU8A36MpL99s70VzHHAecHEp5dk2cxWwHrgiyWuSvAv4OPD5zni/ArwqyflJZiX5CPBO4MJ+\n5kaSJA1Wv4efPkxz6Oafepa/n6a8UEq5IMleNPeU2Qf4IXB8KWV9J78IeB64BpgGfBf4aM9rvhu4\nmObKpA1t9oyxlaWUDUlOBL5MsxdoHfA14JxOZm2SBcAlwO3ACLC4lHJ5J/NAkhOAJTSF52GaS8B7\nr4iSJEk7sX7vU7NVe3ZKKYuBxZtZ/wxwevvYVOZxmvvMbG47PwVO3ELmLuCNW8gso7nXjiRJ2kX5\n3U+SJKkKlhpJklQFS40kSaqCpUaSJFXBUiNJkqpgqZEkSVWw1EiSpCpYaiRJUhUsNZIkqQqWGkmS\nVAVLjSRJqoKlRpIkVcFSI0mSqmCpkSRJVbDUSJKkKlhqJElSFSw1kiSpCpYaSZJUBUuNJEmqgqVG\nkiRVwVIjSZKqYKmRJElVsNRIkqQqWGokSVIVLDWSJKkKlhpJklQFS40kSaqCpUaSJFXBUiNJkqpg\nqZEkSVWw1EiSpCpYaiRJUhUsNZIkqQqWGkmSVAVLjSRJqoKlRpIkVcFSI0mSqmCpkSRJVbDUSJKk\nKlhqJElSFSw1kiSpCpYaSZJUBUuNJEmqgqVGkiRVwVIjSZKqYKmRJElVsNRIkqQqWGokSVIVLDWS\nJKkKlhpJklQFS40kSapC36UmybFJ/i7Jz5JsSPJH42TOTfJIkieT3JDk4J7105JckmQkyRNJrkmy\nX0/mxUm+lWQ0yWNJLkuyd09m/yTXJVmXZHWSC5JM6ckcmmRZkqeSPJjkzHHG+6Yky5M8neTeJKf0\nOy+SJGmwtmVPzd7APwMfAUrvyiRnAx8DPgQcBawDlibZoxO7CDgBeAcwD3g58O2el7oKmA3Mb7Pz\ngEs725kCXA9MBY4BTgHeB5zbybwIWArcD8wBzgQWJzm1kzkQuBb4PnAY8AXgsiRv2cr5kCRJO4Gp\n/f6FUsp3ge8CJMk4kTOA80op17aZ9wJrgLcDVyeZDnwAOKmU8oM2835gZZKjSim3JZkNHAfMLaXc\n0WZOB65L8slSyup2/auBN5dSRoA7k3wG+Kski0spzwEnA7sDH2yfr0xyOPAJ4LJ2vKcB95VSzmqf\nr0ryBmARcEO/8yNJkgZjQs+pSXIQMJNmrwcApZS1wK3A69pFR9CUqW5mFfBQJ3MM8NhYoWndSLNn\n6OhO5s620IxZCswAXtvJLGsLTTczK8mMTubGnreytDMWSZK0C5joE4Vn0hSPNT3L17TrAIaA9W3Z\n2VRmJvCL7spSyvPAoz2Z8bbDBGWmJ5mGJEnaJfR9+KkS4x0269uiRYuYMWPGRssWLlzIwoULJ+Ll\nJUnapQ0PDzM8PLzRstHR0e22vYkuNatpCsMQG+/9GALu6GT2SDK9Z2/NULtuLNN7NdRuwEt6Mkf2\nbH+os27sz6FxMmUrMmtLKc/8x7f4a0uWLGHOnDmbi0iSNGmN94v+ihUrmDt37nbZ3oQefiql3E9T\nEuaPLWtPDD4auLldtBx4riczCzgAuKVddAuwT3tS75j5NIXp1k7mkCT7djILgFHg7k5mXluIuplV\npZTRTmY+G1vQGYskSdoFbMt9avZOcliS32sXvap9vn/7/CLg00neluQQ4BvAw8B34FcnDl8OXNje\nH2YucAVwUynltjZzD83Jul9NcmSS1wNfBIbbK58AvkdTXr7Z3ovmOOA84OJSyrNt5ipgPXBFktck\neRfwceDznbf0lfY9nJ9kVpKPAO8ELux3biRJ0uBsy+GnI4B/pDmEU/h1Qfg68IFSygVJ9qK5p8w+\nwA+B40sp6zuvsQh4HrgGmEZzifhHe7bzbuBimiuTNrTZM8ZWllI2JDkR+DLNXqB1wNeAczqZtUkW\nAJcAtwMjwOJSyuWdzANJTgCW0BSeh2kuAe+9IkqSJO3EtuU+NT9gC3t4SimLgcWbWf8McHr72FTm\ncZr7zGxuOz8FTtxC5i7gjVvILAO2zwE+SZK0Q/jdT5IkqQqWGkmSVAVLjSRJqoKlRpIkVcFSI0mS\nqmCpkSRJVbDUSJKkKlhqJElSFSw1kiSpCpYaSZJUhW357idJFVm5cuWgh7DT2XfffTnggAMGPQxJ\nfbLUSJPWz4EpnHzyZr9ibVLac8+9WLVqpcVG2sVYaqRJ63FgA3AlMHvAY9mZrOTpp09mZGTEUiPt\nYiw10qQ3G5gz6EFI0gvmicKSJKkKlhpJklQFS40kSaqCpUaSJFXBUiNJkqpgqZEkSVWw1EiSpCpY\naiRJUhUsNZIkqQqWGkmSVAVLjSRJqoKlRpIkVcFSI0mSqmCpkSRJVbDUSJKkKlhqJElSFSw1kiSp\nCpYaSZJUBUuNJEmqgqVGkiRVwVIjSZKqYKmRJElVsNRIkqQqTB30ACRpZ7Ry5cpBD0Gq0vb8t2Wp\nkaSN/ByYwsknnzzogUjqk6VGkjbyOLABuBKYPeCxSDW6HvjMdnllS40kjWs2MGfQg5AqtP0OP3mi\nsCRJqoKlRpIkVcFSI0mSqmCpkSRJVbDUSJKkKlhqJElSFSw1kiSpCpYaSZJUBUuNJEmqgqVGO9Dw\noAewi3Le+uecbRvnrX/O2c7EUtNK8tEk9yd5KsmPkxw56DHVx3/828Z5659ztm2ct/45ZzsTSw2Q\n5F3A54FzgMOBnwBLk+w70IFJkqStZqlpLAIuLaV8o5RyD/Bh4EngA4MdliRJ2lqTvtQk2R2YC3x/\nbFkppQA3Aq8b1LgkSVJ/pg56ADuBfYHdgDU9y9cAszbxd/YEWLlyYr4+/devcz3b8yvZB+9h4FuD\nHsQuaHvN203tnzV+7l7InNU8L1viv9H+OWf9G/s31vwsnUhpdkpMXkleBvwMeF0p5dbO8vOBeaWU\n/7C3Jsm78VMsSdIL8Z5SylUT+YLuqYER4HlgqGf5ELB6E39nKfAe4AHg6e02MkmS6rMncCDNz9IJ\nNen31AAk+TFwaynljPZ5gIeAvy6lfG6gg5MkSVvFPTWNC4GvJVkO3EZzNdRewNcGOShJkrT1LDVA\nKeXq9p4059Icdvpn4LhSyr8PdmSSJGlrefhJkiRVYdLfp0aSJNXBUiNJkqpgqdmMJH+R5KYk65I8\nuonM/kmuazOrk1yQZEpP5tAky9ovy3wwyZk75h3sHJI8kGRD5/F8krN6Mlucx8nGL1ndtCTn9Hym\nNiS5uydzbpJHkjyZ5IYkBw9qvIOS5Ngkf5fkZ+0c/dE4mc3OU5JpSS5JMpLkiSTXJNlvx72LHWtL\nc5bkb8b57F3fk5lsc/apJLclWZtkTZK/TfI74+S2+2dtUv/Q2Aq7A1cDXx5vZftD93qaE66PAU4B\n3kdzwvFY5kU01+LfD8wBzgQWJzl1ew58J1OAT9OchD0TeBnwxbGVWzOPk41fsrpV7uLXn6mZwBvG\nViQ5G/gY8CHgKGAdzfztMYBxDtLeNBc+fITm3+FGtnKeLgJOAN4BzANeDnx7+w57oDY7Z62/Z+PP\n3sKe9ZNtzo6l+T/9aOAPaX52fi/Jb4wFdthnrZTiYwsPmh+yj46z/HjgWWDfzrI/Ax4DprbPT6O5\nwd/UTuazwN2Dfl87cP7uBz6+mfVbnMfJ9gB+DHyh8zw092M/a9Bj2xkeNGVvxWbWPwIs6jyfDjwF\n/Omgxz7AOdsA/FE/89Q+fwb4k05mVvtaRw36PQ1ozv4G+J+b+TuTes7a97tv+37f0Fm2Qz5r7ql5\nYY4B7iyljHSWLQVmAK/tZJaVUp7rycxKMmPHDHOn8OftLsUVST6ZZLfOuq2Zx0nDL1ndar/dHiL4\ntyRXJtkfIMlBNL89d+dvLXArzt+vbOU8HUGzB7WbWUVzc9LJPJdvag+z3JPkS0le0lk3F+dsH5q9\nXI/Cjv2sWWpemJmM/0WYY+u2NlO7LwAnAW8CvgL8BXB+Z71ztLHNfcnqZJyP8fyY5hDlccCHgYOA\nZUn2pplLWW8yAAADbElEQVSjgvO3JVszT0PA+vYH0KYyk83fA+8F/gA4C3gjcH2StOtnMonnrJ2H\ni4AflVLGznPbYZ+1SXfzvSSfBc7eTKQAs0sp9+6gIe2S+pnHUspFneV3JVkPXJrkU6WUZ7frQFWl\nUkr3O2PuSnIb8CDwp8A9gxmVJoNSytWdp/+a5E7g32h+afvHgQxq5/Il4DXA6wex8UlXaoD/RnNM\ndHPu28rXWg30XpEy1Fk39ud4X5bZzeyKXsg83kbz2TsQ+D9s3TxOJtvyJauTWillNMm9wMHAP9Gc\ngzTExr8ZDgF37PjR7bRWs+V5Wg3skWR6z2/QfhZbpZT7k4zQfPb+kUk8Z0kuBt4KHFtK+Xln1Q77\nrE26w0+llF+2ew8293huy68EwC3AIT1XpCwARoG7O5l5PeeQLABWlVJGX/AbGpAXOI+H05z89Yv2\n+dbM46TR7r1aDswfW9bu0p0P3Dyoce3MkvwmzQ+VR0op99P8J9idv+k0V2Y4f62tnKflwHM9mVnA\nATT/bie9JK8AXgqM/RCflHPWFpo/Bt5cSnmou26HftYGfZb0zvwA9gcOA/6S5gfsYe1j73b9FJpL\nbf8eOJTm+P4a4LzOa0ynOev76zS75N4F/D/gg4N+fztoDo8Bzmjn5yDgPe0cXdHJbHEeJ9uD5jDK\nkzTH7l8NXAr8EvitQY9tZ3gAn6O55POVwO8DN7SfmZe2689q5+ttwCHA/6LZK7jHoMe+g+dp7/b/\nrN+j+UXiP7fP99/aeaI5nHA/zeGVucBNwA8H/d4GMWftugtofhi/kuYH8O3ASmD3STxnX6K5WvVY\nmj0rY489O5kd8lkb+GTszA+awyvPj/OY18nsD1zbFpU1NCfATul5nd8FftD+kHoI+OSg39sOnMPD\naVr2ozT3Jbir/XDv3pPb4jxOtgfNfTIeoLns8RbgiEGPaWd5AMM0l7g/1f6bugo4qCezmOYXiidp\nrqY7eNDjHsA8vbH9wdz7f1j3l4rNzhMwjeYeJCPAE8D/APYb9HsbxJwBewLfpdnr8DTNIfYv0/PL\nxiScs/Hm63ngvT257f5Z8wstJUlSFSbdOTWSJKlOlhpJklQFS40kSaqCpUaSJFXBUiNJkqpgqZEk\nSVWw1EiSpCpYaiRJUhUsNZIkqQqWGkmSVAVLjSRJqsL/BwSSiZCt6e2DAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x110e3aac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "buckets = [-87.0, 15, 60, 200]\n",
    "rdd_histogram_data = ml_bucketized_features\\\n",
    "  .select(\"ArrDelay\")\\\n",
    "  .rdd\\\n",
    "  .flatMap(lambda x: x)\\\n",
    "  .histogram(buckets)\n",
    "\n",
    "create_hist(rdd_histogram_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wow, that is hugeloy... distorted! This doesn't look like a good scheme!\n",
    "\n",
    "### Trying To Find Balanced Classes\n",
    "Instead, lets try a bucketing scheme that might result in more balanced bars..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Container object of 6 artists>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAFkCAYAAADYL8pXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XuQX2Wd5/H3h/vImCBm7OgKostMDC64JHIbJeJkhEEY\nZ3a11GBKvNWoKFKxBEdLhyxMlYIrwRG8lICOID3F4jq6EA1CqRkRyZrgLEgIq4INYoI9YMIGwi3P\n/nFO68lvOp100pf0k/er6lTX7zzfPuf5Pb9O9yfPuaWUgiRJ0lS3x2R3QJIkaSwYaiRJUhUMNZIk\nqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFUYVapJ8OMmKJBuSrEvy\n9SR/0lPzpSSbe5alPTX7Jrk0yWCSR5Jcm+Q5PTXPSvLVJOuTPJzksiT799QclOT6JBuTrE1yYZI9\nemqOSLI8yWNJfpnk7GHe1wlJVibZlOTuJKePZlwkSdLkG+1MzfHAZ4BjgD8H9gZuSPIHPXXfAvqA\nme2yoKf9YuAU4HXAPOB5wNd6aq4GZgPz29p5wBeGGtvwshTYCzgWOB14K3Bep+aZwDLgHmAOcDaw\nOMk7OzWHANcBNwEvBT4NXJbk1dsxHpIkaReRnXmgZZIZwIPAvFLKD9p1XwKml1L+61a+ZxrwG+BN\npZSvt+tmAauBY0spK5LMBn4KzC2l3NbWnARcDzy/lLI2ycnAN4HnllIG25p3AZ8A/qiU8lSS9wDn\nAzNLKU+1NR8H/qqUclj7+gLg5FLKEZ0+9rfv4TU7PDiSJGlC7ew5NQcABXioZ/0J7eGpu5J8NsmB\nnba5NLMrNw2tKKWsAQaA49pVxwIPDwWa1o3tvo7p1Nw+FGhay4DpwEs6NcuHAk2nZlaS6Z2aG3v6\nv6zTF0mSNAXstaPfmCQ0h5F+UEq5s9P0LZpDSfcA/xH4OLA0yXGlmRaaCTxRStnQs8l1bRvt1we7\njaWUp5M81FOzbphtDLX9a/v1FyPUrB9hO9OS7FtKeXyY9/5s4CTgXmBTb7skSdqq/YBDgGWllH8b\nyw3vcKgBPgscBry8u7KUck3n5U+T3A78HDgB+O5O7G8sZSe//yTgq2PREUmSdlNvpjl/dszsUKhJ\ncgnwGuD4UsqvR6otpdyTZBA4lCbUrAX2STKtZ7amr22j/dp7NdSewIE9NUf17K6v0zb0tW+YmrId\nNRuGm6Vp3Qtw1VVXMXv27K2UaKwtWrSIJUuWTHY3diuO+cRzzCeeYz6xVq9ezcKFC6H9WzqWRh1q\n2kDzV8ArSykD21H/fODZwFD4WQk8RXNVU/dE4YOBW9qaW4ADkhzZOa9mPs0My62dmo8kmdE5r+ZE\nmkNKd3Zq/j7JnqWUpzs1a0op6zs1J/d0+8ROX4azCWD27NnMmTNnGyOgsTJ9+nTHe4I55hPPMZ94\njvmkGfPTN0Z7n5rP0kwXnQZsTNLXLvu17fu394o5JskLkswH/hm4m+bkW9rZmcuBi9r7w8wFrgBu\nLqWsaGvuauu/mOSoJC+nuZS8v5QyNMNyA014ubK9F81JNFc6XVJKebKtuRp4ArgiyWFJ3gi8H/hU\n5219HnhRkguSzEpyBvB64KLRjI0kSZpco7366d3ANOB7wAOd5Q1t+9PAEcA3gDXAF4H/TXPJ95Od\n7SyiuTfMtZ1tva5nX6cBd9FcmXQdsBx411BjKWUzcGq7zx8CXwG+DJzbqdlAM+tyCPBj4JPA4lLK\n5Z2ae2nug/PnwE/avr2jlNJ7RZQkSdqFjerwUyllxBBUStkE/MV2bOdx4Mx22VrNb4GF29jOfTTB\nZqSaO4BXbqNmOc2l5pIkaYry2U+aMhYs6L0xtcabYz7xHPOJ55jXY6fuKLy7SjIHWLly5UpPLpMk\naRRWrVrF3LlzoXlqwKqx3LYzNZIkqQqGGkmSVAVDjSRJqoKhRpIkVWFnnv0kaYINDAwwODi47cIx\nMmPGDA4++OAJ258k7QxDjTRFDAwMMGvWbDZtenTC9rnffs9gzZrVBhtJU4KhRpoiBgcH20BzFTAR\nD1JdzaZNCxkcHDTUSJoSDDXSlDMb8P5IktTLE4UlSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXB\nUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJ\nVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGok\nSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqG\nGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFUYVapJ8OMmKJBuSrEvy9SR/\nMkzdeUkeSPJoku8kObSnfd8klyYZTPJIkmuTPKen5llJvppkfZKHk1yWZP+emoOSXJ9kY5K1SS5M\nskdPzRFJlid5LMkvk5w9TH9PSLIyyaYkdyc5fTTjIkmSJt9oZ2qOBz4DHAP8ObA3cEOSPxgqSPIh\n4H3A3wBHAxuBZUn26WznYuAU4HXAPOB5wNd69nU1MBuY39bOA77Q2c8ewFJgL+BY4HTgrcB5nZpn\nAsuAe4A5wNnA4iTv7NQcAlwH3AS8FPg0cFmSV49qZCRJ0qTaazTFpZTXdF8neSvwIDAX+EG7+izg\n/FLKdW3NW4B1wF8D1ySZBrwdeFMp5fttzduA1UmOLqWsSDIbOAmYW0q5ra05E7g+yQdLKWvb9hcD\nryqlDAK3J/kY8Ikki0spTwELaYLXO9rXq5McCXwAuKzt73uAX5RSzmlfr0nyCmAR8J3RjI8kSZo8\nO3tOzQFAAR4CSPJCYCbNrAcApZQNwK3Ace2ql9GEqW7NGmCgU3Ms8PBQoGnd2O7rmE7N7W2gGbIM\nmA68pFOzvA003ZpZSaZ3am7seV/LOn2RJElTwA6HmiShOYz0g1LKne3qmTTBY11P+bq2DaAPeKIN\nO1urmUkzA/Q7pZSnacJTt2a4/TBGNdOS7IskSZoSRnX4qcdngcOAl49RXyZSxmIjixYtYvr06Vus\nW7BgAQsWLBiLzUuSNKX19/fT39+/xbr169eP2/52KNQkuQR4DXB8KeXXnaa1NIGhjy1nP/qA2zo1\n+ySZ1jNb09e2DdX0Xg21J3BgT81RPV3r67QNfe0bpqZsR82GUsrjjGDJkiXMmTNnpBJJknZbw/1H\nf9WqVcydO3dc9jfqw09toPkrmhN0B7ptpZR7aELC/E79NJrzYH7YrloJPNVTMws4GLilXXULcEB7\nUu+Q+TSB6dZOzeFJZnRqTgTWA3d2aua1gahbs6aUsr5TM58tndjpiyRJmgJGe5+azwJvBk4DNibp\na5f9OmUXAx9N8pdJDge+AtwPfAN+d+Lw5cBF7f1h5gJXADeXUla0NXfRnKz7xSRHJXk5zaXk/e2V\nTwA30ISXK9t70ZwEnA9cUkp5sq25GngCuCLJYUneCLwf+FSnv58HXpTkgiSzkpwBvB64aDRjI0mS\nJtdoDz+9m+bQzfd61r+NJrxQSrkwyTNo7ilzAPAvwMmllCc69YuAp4FrgX2BbwPv7dnmacAlNFcm\nbW5rzxpqLKVsTnIq8DmaWaCNwJeBczs1G5KcCFwK/BgYBBaXUi7v1Nyb5BRgCU3guZ/mEvDeK6Ik\nSdIubLT3qdmumZ1SymJg8QjtjwNntsvWan5Lc5+ZkfZzH3DqNmruAF65jZrlNPfakSRJU5TPfpIk\nSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhq\nJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkK\nhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJ\nqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAj\nSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw\n1EiSpCoYaiRJUhVGHWqSHJ/km0l+lWRzktf2tH+pXd9dlvbU7Jvk0iSDSR5Jcm2S5/TUPCvJV5Os\nT/JwksuS7N9Tc1CS65NsTLI2yYVJ9uipOSLJ8iSPJfllkrOHeU8nJFmZZFOSu5OcPtpxkSRJk2tH\nZmr2B34CnAGUrdR8C+gDZrbLgp72i4FTgNcB84DnAV/rqbkamA3Mb2vnAV8YamzDy1JgL+BY4HTg\nrcB5nZpnAsuAe4A5wNnA4iTv7NQcAlwH3AS8FPg0cFmSV484CpIkaZey12i/oZTybeDbAEmylbLH\nSym/Ga4hyTTg7cCbSinfb9e9DVid5OhSyooks4GTgLmllNvamjOB65N8sJSytm1/MfCqUsogcHuS\njwGfSLK4lPIUsBDYG3hH+3p1kiOBDwCXtV16D/CLUso57es1SV4BLAK+M9rxkSRJk2O8zqk5Icm6\nJHcl+WySAzttc2nC1E1DK0opa4AB4Lh21bHAw0OBpnUjzczQMZ2a29tAM2QZMB14SadmeRtoujWz\nkkzv1NzY0/9lnb5IkqQpYDxCzbeAtwB/BpwDvBJY2pnVmQk8UUrZ0PN969q2oZoHu42llKeBh3pq\n1g2zDcaoZlqSfYd5f5IkaRc06sNP21JKuabz8qdJbgd+DpwAfHes97eDtnbYbFQWLVrE9OnTt1i3\nYMECFizoPYVIkqTdT39/P/39/VusW79+/bjtb8xDTa9Syj1JBoFDaULNWmCfJNN6Zmv62jbar71X\nQ+0JHNhTc1TP7vo6bUNf+4apKdtRs6GU8vhI723JkiXMmTNnpBJJknZbw/1Hf9WqVcydO3dc9jfu\n96lJ8nzg2cCv21UrgadormoaqpkFHAzc0q66BTigPal3yHyaGZZbOzWHJ5nRqTkRWA/c2amZ1wai\nbs2aUsr6Ts18tnRipy+SJGkK2JH71Oyf5KVJ/nO76kXt64PatguTHJPkBUnmA/8M3E1z8i3t7Mzl\nwEXt/WHmAlcAN5dSVrQ1d7X1X0xyVJKXA58B+tsrnwBuoAkvV7b3ojkJOB+4pJTyZFtzNfAEcEWS\nw5K8EXg/8KnOW/p8+x4uSDIryRnA64GLRjs2kiRp8uzI4aeX0RxGKu0yFBD+kebeNUfQnCh8APAA\nTTj5u07QgOZy6aeBa4F9aS4Rf2/Pfk4DLqG5MmlzW3vWUGMpZXOSU4HPAT8ENgJfBs7t1GxIciJw\nKfBjYBBYXEq5vFNzb5JTgCU0ged+mkvAe6+IkiRJu7AduU/N9xl5hucvtmMbjwNntsvWan5Lc5+Z\nkbZzH3DqNmruoLkCa6Sa5TSXmkuSpCnKZz9JkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaS\nJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCo\nkSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQq\nGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIk\nqQqGGkmSVAVDjSRJqsJek90BSbu21atXT9i+ZsyYwcEHHzxh+5NUF0ONpK34NbAHCxcunLA97rff\nM1izZrXBRtIOMdRI2orfApuBq4DZE7C/1WzatJDBwUFDjaQdYqiRtA2zgTmT3QlJ2iZPFJYkSVUw\n1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKow61CQ5\nPsk3k/wqyeYkrx2m5rwkDyR5NMl3khza075vkkuTDCZ5JMm1SZ7TU/OsJF9Nsj7Jw0kuS7J/T81B\nSa5PsjHJ2iQXJtmjp+aIJMuTPJbkl0nOHqa/JyRZmWRTkruTnD7acZEkSZNrR2Zq9gd+ApwBlN7G\nJB8C3gf8DXA0sBFYlmSfTtnFwCnA64B5wPOAr/Vs6mqah87Mb2vnAV/o7GcPYCnN86uOBU4H3gqc\n16l5JrAMuIfm4TVnA4uTvLNTcwhwHXAT8FLg08BlSV69neMhSZJ2AaN+oGUp5dvAtwGSZJiSs4Dz\nSynXtTVvAdYBfw1ck2Qa8HbgTaWU77c1bwNWJzm6lLIiyWzgJGBuKeW2tuZM4PokHyylrG3bXwy8\nqpQyCNye5GPAJ5IsLqU8BSwE9gbe0b5eneRI4APAZW1/3wP8opRyTvt6TZJXAIuA74x2fCRJ0uQY\n03NqkrwQmEkz6wFAKWUDcCtwXLvqZTRhqluzBhjo1BwLPDwUaFo30swMHdOpub0NNEOWAdOBl3Rq\nlreBplszK8n0Ts2NPW9lWacvkiRpChjrE4Vn0gSPdT3r17VtAH3AE23Y2VrNTODBbmMp5WngoZ6a\n4fbDGNVMS7IvkiRpShj14adKDHfYbNQWLVrE9OnTt1i3YMECFixYMBablyRpSuvv76e/v3+LdevX\nrx+3/Y11qFlLExj62HL2ow+4rVOzT5JpPbM1fW3bUE3v1VB7Agf21BzVs/++TtvQ175hasp21Gwo\npTz+79/i7y1ZsoQ5c+aMVKLKDQwMMDg4uO3CMbB69eoJ2Y8kjZXh/qO/atUq5s6dOy77G9NQU0q5\nJ8lamiuW/g9Ae2LwMcClbdlK4Km25uttzSzgYOCWtuYW4IAkR3bOq5lPE5hu7dR8JMmMznk1JwLr\ngTs7NX+fZM/28NVQzZpSyvpOzck9b+XETl+kYQ0MDDBr1mw2bXp0srsiSWIHQk17r5hD+f0hnBcl\neSnwUCnlPprLtT+a5GfAvcD5wP3AN6A5cTjJ5cBFSR4GHgH+Abi5lLKirbkryTLgi0neA+wDfAbo\nb698AriBJrxc2V5G/tx2X5eUUp5sa64G/g64IskFwOHA+2mu0BryeeC9bfsVNOHp9cBrRjs22r0M\nDg62geYqmrsPjLelwMcmYD+SNDXtyEzNy4Dv0hzCKcCn2vX/CLy9lHJhkmfQ3FPmAOBfgJNLKU90\ntrEIeBq4FtiX5hLx9/bs5zTgEporkza3tb8LI6WUzUlOBT4H/JDmfjhfBs7t1GxIciLNLNGPgUFg\ncSnl8k7NvUlOAZbQBJ77aS4B770iStqK2TS3QRpvHn6SpJHsyH1qvs82rpoqpSwGFo/Q/jhwZrts\nrea3NPeZGWk/9wGnbqPmDuCV26hZDozPAT5JkjQhfPaTJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRI\nkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUM\nNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJU\nBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaS\nJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCo\nkSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqsNdYbzDJucC5Pavv\nKqUc1qk5D3gncABwM/CeUsrPOu37AhcBbwT2BZYBZ5RSHuzUPAu4BDgV2Ax8DTirlLKxU3MQ8Hng\nBOAR4CvA35ZSNndqjmi3cxTwIHBJKeWTOzcKknZ1AwMDDA4OTnY3pGHNmDGDgw8+eLK7MeWMeahp\n3QHMB9K+fmqoIcmHgPcBbwHuBf4eWJZkdinlibbsYuBk4HXABuBSmtByfGcfVwN97X72Ab4MfAFY\n2O5nD2Ap8ABwLPA84ErgCeCjbc0zaQLTDcC7gMOBLyV5uJRy2VgMhKRdz8DAALNmzWbTpkcnuyvS\nsPbb7xmsWbPaYDNK4xVqniql/GYrbWcB55dSrgNI8hZgHfDXwDVJpgFvB95USvl+W/M2YHWSo0sp\nK5LMBk4C5pZSbmtrzgSuT/LBUsratv3FwKtKKYPA7Uk+BnwiyeJSylM0AWhv4B3t69VJjgQ+ABhq\npEoNDg62geYqYPZkd0fqsZpNmxYyODhoqBml8Qo1f5zkV8Am4Bbgw6WU+5K8EJgJ3DRUWErZkORW\n4DjgGuBlbb+6NWuSDLQ1K2hmXh4eCjStG4ECHAN8o625vQ00Q5YBnwNeAvxrW7O8DTTdmnOSTC+l\nrN/5oZC065oNzJnsTkgaI+NxovCPgLfSzJS8G3ghsDzJ/jSBptDMzHSta9ugOaT0RCllwwg1M2nO\nf/mdUsrTwEM9NcPth1HWSJKkKWDMZ2pKKcs6L+9IsgL4JfAG4K6x3t9kWrRoEdOnT99i3YIFC1iw\nYMEk9UiSpF1Hf38//f39W6xbv378DoKM1+Gn3ymlrE9yN3Ao8D2ak4f72HKGpA8YOpS0FtgnybSe\n2Zq+tm2o5jnd/STZEziwp+aonu70ddqGvvZto2arlixZwpw5Tl1LkjSc4f6jv2rVKubOnTsu+xv3\n+9Qk+UOaQPNAKeUemrAwv9M+jeY8mB+2q1bSXC3VrZkFHExzfg7t1wPak3qHDF1tdWun5vAkMzo1\nJwLrgTs7NfPaQNStWeP5NJIkTS1jHmqSfDLJvCQvSPKnwNeBJ4F/aksuBj6a5C+THE5z75j7aU7u\npZ2duRy4KMkJSeYCVwA3l1JWtDV30ZzQ+8UkRyV5OfAZoL+98gmay7TvBK5MckSSk4Dzae5D82Rb\nczXNJd5XJDksyRuB9wOfGutxkSRJ42s8Dj89nyYsPBv4DfAD4NhSyr8BlFIuTPIMmnvKHAD8C3By\n5x41AIuAp4FraW6+923gvT37OY3mpnk30tx871qay8Vp97M5yak0Vzv9ENhIcy+bczs1G5KcSHMf\nnB8Dg8DiUsrlOz0KkiRpQo3HicLbPEu2lLIYWDxC++PAme2ytZrf0t5ob4Sa+2juODxSzR3AK0eq\nkSRJuz6f/SRJkqpgqJEkSVUw1EiSpCoYaiRJUhXG/eZ7kjQaq1evrmIfkiaeoUbSLuLXwB4sXDji\nRY2StFWGGkm7iN/S3HLqKpqnZ4+npcDHxnkfkiaaoUbSLmY2MN7PVPPwk1QjTxSWJElVMNRIkqQq\nGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIk\nqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUON\nJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXB\nUCNJkqpgqJEkSVUw1EiSpCrsNdkd0PAGBgYYHByc7G5oBKtXr57sLkiSOgw1u6CBgQFmzZrNpk2P\nTnZXJEmaMgw1u6DBwcE20FwFzJ7s7mirlgIfm+xOSJJahppd2mxgzmR3Qlvl4SdJ2pV4orAkSaqC\noUaSJFXBUCNJkqpgqJEkSVUw1GgK6Z/sDkgTwJ/zieeY18JQ00ry3iT3JHksyY+SHDXZfVIvf/Fo\nd+DP+cRzzGthqAGSvBH4FHAucCTwr8CyJDMmtWOSJGm7GWoai4AvlFK+Ukq5C3g38Cjw9sntliRJ\n2l67fahJsjcwF7hpaF0ppQA3AsdNVr8kSdLoeEdhmAHsCazrWb8OmLWV79kPxu+Bhr/f7lK8a23X\n/cBXJ7sTHTe3Xyfqc3J/U3Nfo7Wr/ZzvDna1Mb8HqPehuZ33td9YbzvNpMTuK8lzgV8Bx5VSbu2s\nvwCYV0r5d7M1SU5j1/oXIEnSVPPmUsrVY7lBZ2pgEHga6OtZ3wes3cr3LAPeDNwLbBq3nkmSVJ/9\ngENo/paOqd1+pgYgyY+AW0spZ7WvAwwA/1BK+eSkdk6SJG0XZ2oaFwFfTrISWEFzNdQzgC9PZqck\nSdL2M9QApZRr2nvSnEdz2OknwEmllN9Mbs8kSdL28vCTJEmqwm5/nxpJklQHQ40kSaqCoWYEST6S\n5OYkG5M8tJWag5Jc39asTXJhkj16ao5Isrx9WOYvk5w9Me9g6ktyb5LNneXpJOf01GzzM9Do+IDX\n8ZPk3J6f6c1J7uypOS/JA0keTfKdJIdOVn+noiTHJ/lmkl+14/vaYWpGHOMk+ya5NMlgkkeSXJvk\nORP3LqaWbY15ki8N83O/tKdmp8fcX/wj2xu4BvjccI3tH86lNCdcHwucDryV5oTjoZpn0lyLfw8w\nBzgbWJzknePZ8YoU4KM0J3DPBJ4LfGaocXs+A42OD3idEHfw+5/pmcArhhqSfAh4H/A3wNHARprx\n32cS+jlV7U9zwccZNL9DtrCdY3wxcArwOmAe8Dzga+Pb7SltxDFvfYstf+4X9LTv/JiXUly2sdD8\noXxomPUnA08CMzrr3gU8DOzVvn4PzQ3+9urUfBy4c7Lf11RYaMLg+0do3+Zn4DLqMf8R8OnO69Dc\nR/6cye5bDQtNWFw1QvsDwKLO62nAY8AbJrvvU3EBNgOvHc0Yt68fB/5Lp2ZWu62jJ/s97erLVsb8\nS8D/HOF7xmTMnanZOccCt5dSBjvrlgHTgZd0apaXUp7qqZmVZPrEdHPK+9t2OnJVkg8m2bPTtj2f\ngbaTD3idMH/cTtP/PMlVSQ4CSPJCmv/Bdsd/A3Arjv+Y2M4xfhnN7G+3Zg3NTVn9HHbcCUnWJbkr\nyWeTHNhpm8sYjLmhZufMZPgHYQ61bW+Ntu7TwJuAE4DPAx8BLui0O75ja6QHvDqeY+NHNIdITwLe\nDbwQWJ5hBjucAAADH0lEQVRkf5oxLjj+42l7xrgPeKINO1ur0eh8C3gL8GfAOcArgaXtHfyhGded\nHvPd7uZ7ST4OfGiEkgLMLqXcPUFd2u2M5jMopVzcWX9HkieALyT5cCnlyXHtqDQOSind593ckWQF\n8EvgDcBdk9MraXyVUq7pvPxpktuBn9P8h/W7Y7Wf3S7UAP+d5tjeSH6xndtaC/ReFdLXaRv6OtzD\nMrs1u5ud+QxW0PzcHgL8X7bvM9D225EHvGonlFLWJ7kbOBT4Hs05TH1sOZPQB9w28b2r0lq2PcZr\ngX2STOuZOfDfwRgppdyTZJDm5/67jNGY73aHn0op/9bOAIy0PLXtLQFwC3B4z1UhJwLrgTs7NfN6\nzgM5EVhTSlm/029oCtrJz+BImhPHHmxfb89noO3Uzn6tBOYPrWunh+cDP5ysftUsyR/S/GJ/oJRy\nD80v8O74TwOOwfEfE9s5xiuBp3pqZgEH0/zO0U5K8nzg2cCv21VjMua740zNdmtP3jsQeAGwZ5KX\ntk0/K6VsBG6g+cN5ZXuJ4HOB84FLOodGrgb+DrgiyQXA4cD7gbMm7p1MTUmOpflF813gEeBPaR4+\nemUnEG7PZ6DR8QGv4yjJJ4H/RXPI6T8A/43mCr5/aksuBj6a5GfAvTQ/z/cD35jwzk5R7flJh9LM\nyAC8qP39/VAp5T62McallA1JLgcuSvIwze+ffwBuLqWsmNA3M0WMNObtci7N5dlr27oLgLtpLuwY\nuzGf7Eu/duWF5hDJ08Ms8zo1BwHXAf+PZirzAmCPnu38J+D7wKM0Z3J/cLLf21RYaGZlbmn/QWyk\nubfHOcDePXXb/AxcRj32Z9D8sn+s/QxeNtl9qmUB+mn+gD7W/j64GnhhT81imsuOH21/6R862f2e\nSgvNSaibh/ndfcX2jjGwL809sQbbP7D/A3jOZL+3XXUZacyB/YBv0wSaTTSnF3wO+KOxHnMfaClJ\nkqqw251TI0mS6mSokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKq\nYKiRJElVMNRIkqQq/H9wRKmYJvYgnwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10cf94b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "buckets = [-87.0, -30, -15, 0, 15, 30, 120]\n",
    "rdd_histogram_data = ml_bucketized_features\\\n",
    "  .select(\"ArrDelay\")\\\n",
    "  .rdd\\\n",
    "  .flatMap(lambda x: x)\\\n",
    "  .histogram(buckets)\n",
    "\n",
    "create_hist(rdd_histogram_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hmmmm lets try again."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Container object of 5 artists>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAFkCAYAAADYL8pXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3X+QXlWd5/H3h98jY4KYsaMriC4zMbjgmMivUSJORhiE\ncWZXSw2mxF81KopULMFxSocsTJWCK8ER/FECOoL0FIvr6AIahFIzIpI1wSmQEHY0GBET7AETNxBC\nyNk/7m29eabT6U76R/rk/aq61fXc8+3nnud05+lPzr3nPimlIEmSNNXtM9kdkCRJGguGGkmSVAVD\njSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUhVGFmiQfSrI8ycYk\n65N8Nckf9dR8Icm2nu3mnpoDk1yRZCDJb5LckORZPTXPSPLlJBuSPJrkyiQH99QcluSmJJuSrEty\nSZJ9emqOSbIsyeNJfpbkvCFe18lJViTZnOT+JGeNZlwkSdLkG+1MzUnAp4DjgT8D9gduSfJ7PXXf\nAPqAme22oKf9MuB04LXAPOA5wFd6aq4DZgPz29p5wOcGG9vwcjOwH3ACcBbwFuDCTs3TgaXAGmAO\ncB6wOMk7OjVHADcCtwEvBj4JXJnkVSMYD0mStIfI7nygZZIZwMPAvFLK99p9XwCml1L+2w6+Zxrw\nK+CNpZSvtvtmAauAE0opy5PMBn4MzC2l3NXWnArcBDy3lLIuyWnA14Fnl1IG2pp3Ah8D/qCUsjXJ\nu4GLgJmllK1tzUeBvyylHNU+vhg4rZRyTKeP/e1rePUuD44kSZpQu3tNzSFAAR7p2X9ye3rqviSf\nTnJop20uzezKbYM7SimrgbXAie2uE4BHBwNN69b2WMd3au4eDDStpcB04EWdmmWDgaZTMyvJ9E7N\nrT39X9rpiyRJmgL229VvTBKa00jfK6Xc22n6Bs2ppDXAfwY+Ctyc5MTSTAvNBLaUUjb2POX6to32\n68PdxlLKU0ke6alZP8RzDLb9a/v1p8PUbBjmeaYlObCU8sQQr/2ZwKnAA8Dm3nZJkrRDBwFHAEtL\nKf8+lk+8y6EG+DRwFPCy7s5SyvWdhz9OcjfwE+Bk4Nu7cbyxlN38/lOBL49FRyRJ2ku9ieb62TGz\nS6EmyeXAq4GTSim/HK62lLImyQBwJE2oWQcckGRaz2xNX9tG+7V3NdS+wKE9Ncf2HK6v0zb4tW+I\nmjKCmo1DzdK0HgC49tprmT179g5KNNYWLVrEkiVLJrsbexXHfOI55hPPMZ9Yq1atYuHChdD+LR1L\now41baD5S+AVpZS1I6h/LvBMYDD8rAC20qxq6l4ofDhwR1tzB3BIkpd0rquZTzPDcmen5m+TzOhc\nV3MKzSmlezs1f59k31LKU52a1aWUDZ2a03q6fUqnL0PZDDB79mzmzJmzkxHQWJk+fbrjPcEc84nn\nmE88x3zSjPnlG6O9T82naaaLzgQ2Jelrt4Pa9oPbe8Ucn+R5SeYD/wzcT3PxLe3szFXApe39YeYC\nVwO3l1KWtzX3tfWfT3JskpfRLCXvL6UMzrDcQhNermnvRXMqzUqny0spT7Y11wFbgKuTHJXkDcD7\ngE90XtZngRckuTjJrCRnA68DLh3N2EiSpMk12tVP7wKmAd8BHupsr2/bnwKOAb4GrAY+D/wfmiXf\nT3aeZxHNvWFu6DzXa3uOdSZwH83KpBuBZcA7BxtLKduAM9pjfh/4EvBF4IJOzUaaWZcjgB8CHwcW\nl1Ku6tQ8QHMfnD8DftT27e2llN4VUZIkaQ82qtNPpZRhQ1ApZTPw5yN4nieAc9ptRzW/Bhbu5Hl+\nThNshqu5B3jFTmqW0Sw1lyRJU5Sf/aQpY8GC3htTa7w55hPPMZ94jnk9duuOwnurJHOAFStWrPDi\nMkmSRmHlypXMnTsXmk8NWDmWz+1MjSRJqoKhRpIkVcFQI0mSqmCokSRJVdidz36SNMHWrl3LwMDA\nzgvHyIwZMzj88MMn7HiStDsMNdIUsXbtWmbNms3mzY9N2DEPOuhprF69ymAjaUow1EhTxMDAQBto\nrgUm4oNUV7F580IGBgYMNZKmBEONNOXMBrw/kiT18kJhSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiR\nJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoY\naiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSp\nCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40k\nSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVIVRhZokH0qyPMnGJOuT\nfDXJHw1Rd2GSh5I8luRbSY7saT8wyRVJBpL8JskNSZ7VU/OMJF9OsiHJo0muTHJwT81hSW5KsinJ\nuiSXJNmnp+aYJMuSPJ7kZ0nOG6K/JydZkWRzkvuTnDWacZEkSZNvtDM1JwGfAo4H/gzYH7glye8N\nFiT5IPBe4K+B44BNwNIkB3Se5zLgdOC1wDzgOcBXeo51HTAbmN/WzgM+1znOPsDNwH7ACcBZwFuA\nCzs1TweWAmuAOcB5wOIk7+jUHAHcCNwGvBj4JHBlkleNamQkSdKk2m80xaWUV3cfJ3kL8DAwF/he\nu/tc4KJSyo1tzZuB9cBfAdcnmQa8DXhjKeW7bc1bgVVJjiulLE8yGzgVmFtKuautOQe4KckHSinr\n2vYXAq8spQwAdyf5CPCxJItLKVuBhTTB6+3t41VJXgK8H7iy7e+7gZ+WUs5vH69O8nJgEfCt0YyP\nJEmaPLt7Tc0hQAEeAUjyfGAmzawHAKWUjcCdwIntrpfShKluzWpgbafmBODRwUDTurU91vGdmrvb\nQDNoKTAdeFGnZlkbaLo1s5JM79Tc2vO6lnb6IkmSpoBdDjVJQnMa6XullHvb3TNpgsf6nvL1bRtA\nH7ClDTs7qplJMwP0W6WUp2jCU7dmqOMwRjXTkhyIJEmaEkZ1+qnHp4GjgJeNUV8mUsbiSRYtWsT0\n6dO327dgwQIWLFgwFk8vSdKU1t/fT39//3b7NmzYMG7H26VQk+Ry4NXASaWUX3aa1tEEhj62n/3o\nA+7q1ByQZFrPbE1f2zZY07saal/g0J6aY3u61tdpG/zaN0RNGUHNxlLKEwxjyZIlzJkzZ7gSSZL2\nWkP9R3/lypXMnTt3XI436tNPbaD5S5oLdNd220opa2hCwvxO/TSa62C+3+5aAWztqZkFHA7c0e66\nAzikvah30HyawHRnp+boJDM6NacAG4B7OzXz2kDUrVldStnQqZnP9k7p9EWSJE0Bo71PzaeBNwFn\nApuS9LXbQZ2yy4APJ/mLJEcDXwIeBL4Gv71w+Crg0vb+MHOBq4HbSynL25r7aC7W/XySY5O8jGYp\neX+78gngFprwck17L5pTgYuAy0spT7Y11wFbgKuTHJXkDcD7gE90+vtZ4AVJLk4yK8nZwOuAS0cz\nNpIkaXKN9vTTu2hO3XynZ/9bacILpZRLkjyN5p4yhwD/ApxWStnSqV8EPAXcABwIfBN4T89znglc\nTrMyaVtbe+5gYyllW5IzgM/QzAJtAr4IXNCp2ZjkFOAK4IfAALC4lHJVp+aBJKcDS2gCz4M0S8B7\nV0RJkqQ92GjvUzOimZ1SymJg8TDtTwDntNuOan5Nc5+Z4Y7zc+CMndTcA7xiJzXLaO61I0mSpig/\n+0mSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mS\nqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRI\nkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUM\nNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJU\nBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaS\nJFXBUCNJkqpgqJEkSVUYdahJclKSryf5RZJtSV7T0/6Fdn93u7mn5sAkVyQZSPKbJDckeVZPzTOS\nfDnJhiSPJrkyycE9NYcluSnJpiTrklySZJ+emmOSLEvyeJKfJTlviNd0cpIVSTYnuT/JWaMdF0mS\nNLl2ZabmYOBHwNlA2UHNN4A+YGa7Lehpvww4HXgtMA94DvCVnprrgNnA/LZ2HvC5wcY2vNwM7Aec\nAJwFvAW4sFPzdGApsAaYA5wHLE7yjk7NEcCNwG3Ai4FPAlcmedWwoyBJkvYo+432G0op3wS+CZAk\nOyh7opTyq6EakkwD3ga8sZTy3XbfW4FVSY4rpSxPMhs4FZhbSrmrrTkHuCnJB0op69r2FwKvLKUM\nAHcn+QjwsSSLSylbgYXA/sDb28erkrwEeD9wZduldwM/LaWc3z5eneTlwCLgW6MdH0mSNDnG65qa\nk5OsT3Jfkk8nObTTNpcmTN02uKOUshpYC5zY7joBeHQw0LRupZkZOr5Tc3cbaAYtBaYDL+rULGsD\nTbdmVpLpnZpbe/q/tNMXSZI0BYxHqPkG8GbgT4HzgVcAN3dmdWYCW0opG3u+b33bNljzcLexlPIU\n8EhPzfohnoMxqpmW5MAhXp8kSdoDjfr0086UUq7vPPxxkruBnwAnA98e6+Ptoh2dNhuVRYsWMX36\n9O32LViwgAULei8hkiRp79Pf309/f/92+zZs2DBuxxvzUNOrlLImyQBwJE2oWQcckGRaz2xNX9tG\n+7V3NdS+wKE9Ncf2HK6v0zb4tW+ImjKCmo2llCeGe21Llixhzpw5w5VIkrTXGuo/+itXrmTu3Lnj\ncrxxv09NkucCzwR+2e5aAWylWdU0WDMLOBy4o911B3BIe1HvoPk0Myx3dmqOTjKjU3MKsAG4t1Mz\nrw1E3ZrVpZQNnZr5bO+UTl8kSdIUsCv3qTk4yYuT/HG76wXt48PatkuSHJ/keUnmA/8M3E9z8S3t\n7MxVwKXt/WHmAlcDt5dSlrc197X1n09ybJKXAZ8C+tuVTwC30ISXa9p70ZwKXARcXkp5sq25DtgC\nXJ3kqCRvAN4HfKLzkj7bvoaLk8xKcjbwOuDS0Y6NJEmaPLty+umlNKeRSrsNBoR/pLl3zTE0Fwof\nAjxEE07+rhM0oFku/RRwA3AgzRLx9/Qc50zgcpqVSdva2nMHG0sp25KcAXwG+D6wCfgicEGnZmOS\nU4ArgB8CA8DiUspVnZoHkpwOLKEJPA/SLAHvXRElSZL2YLtyn5rvMvwMz5+P4DmeAM5ptx3V/Jrm\nPjPDPc/PgTN2UnMPzQqs4WqW0Sw1lyRJU5Sf/SRJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqG\nGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmq\ngqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJ\nkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDU\nSJKkKhhqJElSFQw1kiSpCvtNdgck7dlWrVo1YceaMWMGhx9++IQdT1JdDDWSduCXwD4sXLhwwo54\n0EFPY/XqVQYbSbvEUCNpB34NbAOuBWZPwPFWsXnzQgYGBgw1knaJoUbSTswG5kx2JyRpp7xQWJIk\nVcFQI0mSquDppz3U2rVrGRgYmOxuaA8ykauQJGkqMtTsgdauXcusWbPZvPmxye6KJElThqFmDzQw\nMNAGmoladaKp4WbgI5PdCUnaYxlq9miuOlGXp58kaTheKCxJkqpgqJEkSVUw1EiSpCoYaiRJUhVG\nHWqSnJTk60l+kWRbktcMUXNhkoeSPJbkW0mO7Gk/MMkVSQaS/CbJDUme1VPzjCRfTrIhyaNJrkxy\ncE/NYUluSrIpyboklyTZp6fmmCTLkjye5GdJzhuivycnWZFkc5L7k5w12nGRJEmTa1dmag4GfgSc\nDZTexiQfBN4L/DVwHLAJWJrkgE7ZZcDpwGuBecBzgK/0PNV1NMt/5re184DPdY6zD80a1/2AE4Cz\ngLcAF3Zqng4sBdbQLCM6D1ic5B2dmiOAG4HbgBcDnwSuTPKqEY6HJEnaA4x6SXcp5ZvANwGSZIiS\nc4GLSik3tjVvBtYDfwVcn2Qa8DbgjaWU77Y1bwVWJTmulLI8yWzgVGBuKeWutuYc4KYkHyilrGvb\nXwi8spQyANyd5CPAx5IsLqVsBRYC+wNvbx+vSvIS4P3AlW1/3w38tJRyfvt4dZKXA4uAb412fCRJ\n0uQY02tqkjwfmEkz6wFAKWUjcCdwYrvrpTRhqluzGljbqTkBeHQw0LRupZkZOr5Tc3cbaAYtBaYD\nL+rULGsDTbdmVpLpnZpbe17K0k5fJEnSFDDWFwrPpAke63v2r2/bAPqALW3Y2VHNTODhbmMp5Sng\nkZ6aoY7DGNVMS3IgkiRpSthb7yg81GmzUVu0aBHTp0/fbt+CBQtYsGDBWDy9JElTWn9/P/39/dvt\n27Bhw7gdb6xDzTqawNDH9rMffcBdnZoDkkzrma3pa9sGa3pXQ+0LHNpTc2zP8fs6bYNf+4aoKSOo\n2VhKeeI/vsTfWbJkCXPm+DEGkiQNZaj/6K9cuZK5c+eOy/HG9PRTKWUNTUiYP7ivvTD4eOD77a4V\nwNaemlnA4cAd7a47gEPai3oHzacJTHd2ao5OMqNTcwqwAbi3UzOvDUTdmtWllA2dmvls75ROXyRJ\n0hSwK/epOTjJi5P8cbvrBe3jw9rHlwEfTvIXSY4GvgQ8CHwNfnvh8FXApe39YeYCVwO3l1KWtzX3\n0Vys+/kkxyZ5GfApoL9d+QRwC014uaa9F82pwEXA5aWUJ9ua64AtwNVJjkryBuB9wCc6L+mz7Wu4\nOMmsJGcDrwMuHe3YSJKkybMrp59eCnyb5hRO4XcB4R+Bt5VSLknyNJp7yhwC/AtwWillS+c5FgFP\nATcAB9IsEX9Pz3HOBC6nWZm0ra09d7CxlLItyRnAZ2hmgTYBXwQu6NRsTHIKcAXwQ2AAWFxKuapT\n80CS04ElNIHnQZol4L0roiRJ0h5sV+5T8112MsNTSlkMLB6m/QngnHbbUc2vae4zM9xxfg6csZOa\ne4BX7KRmGTA+J/gkSdKE8LOfJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElV\nMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJ\nUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYa\nSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqC\noUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mS\nqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUhf3G+gmTXABc0LP7vlLKUZ2aC4F3AIcAtwPv\nLqX8W6f9QOBS4A3AgcBS4OxSysOdmmcAlwNnANuArwDnllI2dWoOAz4LnAz8BvgS8DellG2dmmPa\n5zkWeBi4vJTy8d0bBUl7urVr1zIwMDDZ3ZCGNGPGDA4//PDJ7saUM+ahpnUPMB9I+3jrYEOSDwLv\nBd4MPAD8PbA0yexSypa27DLgNOC1wEbgCprQclLnGNcBfe1xDgC+CHwOWNgeZx/gZuAh4ATgOcA1\nwBbgw23N02kC0y3AO4GjgS8kebSUcuVYDISkPc/atWuZNWs2mzc/NtldkYZ00EFPY/XqVQabURqv\nULO1lPKrHbSdC1xUSrkRIMmbgfXAXwHXJ5kGvA14Yynlu23NW4FVSY4rpSxPMhs4FZhbSrmrrTkH\nuCnJB0op69r2FwKvLKUMAHcn+QjwsSSLSylbaQLQ/sDb28erkrwEeD9gqJEqNTAw0Aaaa4HZk90d\nqccqNm9eyMDAgKFmlMYr1Pxhkl8Am4E7gA+VUn6e5PnATOC2wcJSysYkdwInAtcDL2371a1ZnWRt\nW7OcZubl0cFA07oVKMDxwNfamrvbQDNoKfAZ4EXAv7Y1y9pA0605P8n0UsqG3R8KSXuu2cCcye6E\npDEyHhcK/wB4C81MybuA5wPLkhxME2gKzcxM1/q2DZpTSltKKRuHqZlJc/3Lb5VSngIe6akZ6jiM\nskaSJE0BYz5TU0pZ2nl4T5LlwM+A1wP3jfXxJtOiRYuYPn36dvsWLFjAggULJqlHkiTtOfr7++nv\n799u34YN43cSZLxOP/1WKWVDkvuBI4Hv0Fw83Mf2MyR9wOCppHXAAUmm9czW9LVtgzXP6h4nyb7A\noT01x/Z0p6/TNvi1byc1O7RkyRLmzHHqWpKkoQz1H/2VK1cyd+7ccTneuN+nJsnv0wSah0opa2jC\nwvxO+zSa62C+3+5aQbNaqlszCzic5voc2q+HtBf1DhpcbXVnp+boJDM6NacAG4B7OzXz2kDUrVnt\n9TSSJE0tYx5qknw8ybwkz0vyJ8BXgSeBf2pLLgM+nOQvkhxNc++YB2ku7qWdnbkKuDTJyUnmAlcD\nt5dSlrc199Fc0Pv5JMcmeRnwKaC/XfkEzTLte4FrkhyT5FTgIpr70DzZ1lxHs8T76iRHJXkD8D7g\nE2M9LpIkaXyNx+mn59KEhWcCvwK+B5xQSvl3gFLKJUmeRnNPmUOAfwFO69yjBmAR8BRwA83N974J\nvKfnOGfS3DTvVpqb791As1yc9jjbkpxBs9rp+8AmmnvZXNCp2ZjkFJr74PwQGAAWl1Ku2u1RkCRJ\nE2o8LhTe6VWypZTFwOJh2p8Azmm3HdX8mvZGe8PU/JzmjsPD1dwDvGK4GkmStOfzs58kSVIVDDWS\nJKkKhhpJklQFQ40kSarCuN98T5JGY9WqVVUcQ9LEM9RI2kP8EtiHhQuHXdQoSTtkqJG0h/g1zS2n\nrqX59OzxdDPwkXE+hqSJZqiRtIeZDYz3Z6p5+kmqkRcKS5KkKhhqJElSFQw1kiSpCoYaSZJUBUON\nJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXB\nUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJ\nVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGok\nSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZpC+ie7\nA9IE8Pd84jnmtTDUtJK8J8maJI8n+UGSYye7T+rlG4/2Bv6eTzzHvBaGGiDJG4BPABcALwH+FVia\nZMakdkySJI2YoaaxCPhcKeVLpZT7gHcBjwFvm9xuSZKkkdrrQ02S/YG5wG2D+0opBbgVOHGy+iVJ\nkkZnv8nuwB5gBrAvsL5n/3pg1g6+5yCAVatWjUuHfve8NwPjc4yp6UHgy5PdiUl0e/t1on4vaj7e\nRL+20djbf88nw5425muA8fsbM9k6r+ugsX7uNJMSe68kzwZ+AZxYSrmzs/9iYF4p5T/M1iQ5kz3r\nX4AkSVPNm0op143lEzpTAwPAU0Bfz/4+YN0Ovmcp8CbgAWDzuPVMkqT6HAQcQfO3dEzt9TM1AEl+\nANxZSjm3fRxgLfAPpZSPT2rnJEnSiDhT07gU+GKSFcBymtVQTwO+OJmdkiRJI2eoAUop17f3pLmQ\n5rTTj4BTSym/mtyeSZKkkfL0kyRJqsJef58aSZJUB0ONJEmqgqFmGEn+NsntSTYleWQHNYcluamt\nWZfkkiT79NQck2RZ+2GZP0ty3sS8gqkvyQNJtnW2p5Kc31Oz05+BRscPeB0/SS7o+Z3eluTenpoL\nkzyU5LEk30py5GT1dypKclKSryf5RTu+rxmiZtgxTnJgkiuSDCT5TZIbkjxr4l7F1LKzMU/yhSF+\n72/uqdntMfeNf3j7A9cDnxmqsf3DeTPNBdcnAGcBb6G54Hiw5uk0a/HXAHOA84DFSd4xnh2vSAE+\nTHMB90zg2cCnBhtH8jPQ6PgBrxPiHn73Oz0TePlgQ5IPAu8F/ho4DthEM/4HTEI/p6qDaRZ8nE3z\nHrKdEY7xZcDpwGuBecBzgK+Mb7entGHHvPUNtv+9X9DTvvtjXkpx28lG84fykSH2nwY8Cczo7Hsn\n8CiwX/v43TQ3+NuvU/NR4N7Jfl1TYaMJg+8bpn2nPwO3UY/5D4BPdh6H5j7y509232rYaMLiymHa\nHwIWdR5PAx4HXj/ZfZ+KG7ANeM1oxrh9/ATwXzs1s9rnOm6yX9Oevu1gzL8A/K9hvmdMxtyZmt1z\nAnB3KWWgs28pMB14UadmWSlla0/NrCTTJ6abU97ftNORK5N8IMm+nbaR/Aw0Qn7A64T5w3aa/idJ\nrk1yGECS59P8D7Y7/huBO3H8x8QIx/ilNLO/3ZrVNDdl9eew605Osj7JfUk+neTQTttcxmDMDTW7\nZyZDfxDmYNtIa7RjnwTeCJwMfBb4W+DiTrvjO7aG+4BXx3Ns/IDmFOmpwLuA5wPLkhxMM8YFx388\njWSM+4AtbdjZUY1G5xvAm4E/Bc4HXgHc3N7BH5px3e0x3+tuvpfko8AHhykpwOxSyv0T1KW9zmh+\nBqWUyzpgN06QAAAC2ElEQVT770myBfhckg+VUp4c145K46CU0v28m3uSLAd+BrweuG9yeiWNr1LK\n9Z2HP05yN/ATmv+wfnusjrPXhRrgf9Cc2xvOT0f4XOuA3lUhfZ22wa9DfVhmt2Zvszs/g+U0v7dH\nAP+Xkf0MNHK78gGv2g2llA1J7geOBL5Dcw1TH9vPJPQBd01876q0jp2P8TrggCTTemYO/HcwRkop\na5IM0Pzef5sxGvO97vRTKeXf2xmA4batO38mAO4Aju5ZFXIKsAG4t1Mzr+c6kFOA1aWUDbv9gqag\n3fwZvITmwrGH28cj+RlohNrZrxXA/MF97fTwfOD7k9WvmiX5fZo39odKKWto3sC74z8NOB7Hf0yM\ncIxXAFt7amYBh9O852g3JXku8Ezgl+2uMRnzvXGmZsTai/cOBZ4H7JvkxW3Tv5VSNgG30PzhvKZd\nIvhs4CLg8s6pkeuAvwOuTnIxcDTwPuDciXslU1OSE2jeaL4N/Ab4E5oPH72mEwhH8jPQ6PgBr+Mo\nyceB/01zyuk/Af+dZgXfP7UllwEfTvJvwAM0v88PAl+b8M5OUe31SUfSzMgAvKB9/36klPJzdjLG\npZSNSa4CLk3yKM37zz8At5dSlk/oi5kihhvzdruAZnn2urbuYuB+moUdYzfmk730a0/eaE6RPDXE\nNq9TcxhwI/D/aKYyLwb26Xme/wJ8F3iM5kruD0z2a5sKG82szB3tP4hNNPf2OB/Yv6dupz8Dt1GP\n/dk0b/aPtz+Dl052n2rZgH6aP6CPt+8H1wHP76lZTLPs+LH2Tf/Iye73VNpoLkLdNsR799UjHWPg\nQJp7Yg20f2D/J/CsyX5te+o23JgDBwHfpAk0m2kuL/gM8AdjPeZ+oKUkSarCXndNjSRJqpOhRpIk\nVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKq8P8B\n6FJ76lc2QtIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1110af588>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "buckets = [-87.0, -15, 0, 15, 30, 120]\n",
    "rdd_histogram_data = ml_bucketized_features\\\n",
    "  .select(\"ArrDelay\")\\\n",
    "  .rdd\\\n",
    "  .flatMap(lambda x: x)\\\n",
    "  .histogram(buckets)\n",
    "\n",
    "create_hist(rdd_histogram_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets knock out our smallest bucket and try again..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Container object of 4 artists>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjUAAAFkCAYAAADYL8pXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3X+QXlWd5/H3h98jY4KYsaMriC4zMbjgmJZfowScjGEQ\nZsZdLTGYEn/VqChSsQTHKR2ysFsKrgRH8EcJ6AjSUyyuowtIEErNiEhWglMgIewoGBET7AETNxh+\n5ewf97bePNPpdCf9I33yflU91fXc8+17z3O68/Qn595zn5RSkCRJmu72mOoOSJIkjQdDjSRJqoKh\nRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqwphCTZIPJVmZZGOS\n9Um+muSPemq+kGRLz+OGnpp9k1yaZDDJr5Ncm+Q5PTXPSvLlJBuSPJrksiT799QclOT6JJuSrEty\nYZI9emqOSLIiyW+S/DTJ2cO8rhOS3JFkc5L7kpw+lnGRJElTb6wzNccBnwKOBv4M2Bu4Kcnv9dR9\nA+gDZrePRT3tFwMnA68D5gPPA77SU3M1MBdY0NbOBz431NiGlxuAvYBjgNOBtwDndWqeCSwH7gfm\nAWcDS5O8o1NzCHAdcAvwUuCTwGVJXj2K8ZAkSbuI7MwHWiaZBTwMzC+lfLfd9gVgZinlv2zje2YA\nvwTeWEr5arttDrAaOKaUsjLJXOBHQH8p5c625kTgeuD5pZR1SU4Cvg48t5Qy2Na8E/gY8AellKeS\nvBs4H5hdSnmqrfko8FellMPa5xcAJ5VSjuj0caB9Da/Z4cGRJEmTamevqTkAKMAjPdtPaE9P3Zvk\n00kO7LT108yu3DK0oZSyBlgLHNtuOgZ4dCjQtG5uj3V0p+auoUDTWg7MBF7SqVkxFGg6NXOSzOzU\n3NzT/+WdvkiSpGlgrx39xiShOY303VLKPZ2mb9CcSrof+I/AR4Ebkhxbmmmh2cATpZSNPbtc37bR\nfn2421hKeTrJIz0164fZx1Dbv7RffzJCzYYR9jMjyb6llMeHee3PBk4EHgA297ZLkqRt2g84BFhe\nSvm38dzxDoca4NPAYcAruhtLKdd0nv4oyV3Aj4ETgG/txPHGU3by+08EvjweHZEkaTf1JprrZ8fN\nDoWaJJcArwGOK6X8YqTaUsr9SQaBQ2lCzTpgnyQzemZr+to22q+9q6H2BA7sqTmy53B9nbahr33D\n1JRR1Gwcbpam9QDAVVddxdy5c7dRovG2ZMkSli1bNtXd2K045pPPMZ98jvnkWr16NYsXL4b2b+l4\nGnOoaQPNXwHHl1LWjqL++cCzgaHwcwfwFM2qpu6FwgcDt7U1twEHJHlZ57qaBTQzLLd3av42yazO\ndTULaU4p3dOp+W9J9iylPN2pWVNK2dCpOamn2ws7fRnOZoC5c+cyb9687YyAxsvMmTMd70nmmE8+\nx3zyOeZTZtwv3xjrfWo+TTNddBqwKUlf+9ivbd+/vVfM0UlekGQB8E/AfTQX39LOzlwOXNTeH6Yf\nuAK4tZSysq25t63/fJIjk7yCZin5QCllaIblJprwcmV7L5oTaVY6XVJKebKtuRp4ArgiyWFJTgXe\nB3yi87I+C7woyQVJ5iQ5A3g9cNFYxkaSJE2tsa5+ehcwA/g28FDn8Ya2/WngCOBrwBrg88D/oVny\n/WRnP0to7g1zbWdfr+s51mnAvTQrk64DVgDvHGospWwBTmmP+T3gS8AXgXM7NRtpZl0OAX4AfBxY\nWkq5vFPzAM19cP4M+GHbt7eXUnpXREmSpF3YmE4/lVJGDEGllM3An49iP48DZ7aPbdX8Cli8nf38\njCbYjFRzN3D8dmpW0Cw1lyRJ05Sf/aRpY9Gi3htTa6I55pPPMZ98jnk9duqOwrurJPOAO+644w4v\nLpMkaQxWrVpFf38/NJ8asGo89+1MjSRJqoKhRpIkVcFQI0mSqmCokSRJVdiZz36SNMnWrl3L4ODg\n9gvHyaxZszj44IMn7XiStDMMNdI0sXbtWubMmcvmzY9N2jH32+8ZrFmz2mAjaVow1EjTxODgYBto\nrgIm44NUV7N582IGBwcNNZKmBUONNO3MBbw/kiT18kJhSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiR\nJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoY\naiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSp\nCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40k\nSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVIUxhZokH0qyMsnGJOuT\nfDXJHw1Td16Sh5I8luSbSQ7tad83yaVJBpP8Osm1SZ7TU/OsJF9OsiHJo0kuS7J/T81BSa5PsinJ\nuiQXJtmjp+aIJCuS/CbJT5OcPUx/T0hyR5LNSe5LcvpYxkWSJE29sc7UHAd8Cjga+DNgb+CmJL83\nVJDkg8B7gb8GjgI2AcuT7NPZz8XAycDrgPnA84Cv9BzramAusKCtnQ98rnOcPYAbgL2AY4DTgbcA\n53VqngksB+4H5gFnA0uTvKNTcwhwHXAL8FLgk8BlSV49ppGRJElTaq+xFJdSXtN9nuQtwMNAP/Dd\ndvNZwPmllOvamjcD64HXAtckmQG8DXhjKeU7bc1bgdVJjiqlrEwyFzgR6C+l3NnWnAlcn+QDpZR1\nbfuLgVeVUgaBu5J8BPhYkqWllKeAxTTB6+3t89VJXga8H7is7e+7gZ+UUs5pn69J8kpgCfDNsYyP\nJEmaOjt7Tc0BQAEeAUjyQmA2zawHAKWUjcDtwLHtppfThKluzRpgbafmGODRoUDTurk91tGdmrva\nQDNkOTATeEmnZkUbaLo1c5LM7NTc3PO6lnf6IkmSpoEdDjVJQnMa6bullHvazbNpgsf6nvL1bRtA\nH/BEG3a2VTObZgbot0opT9OEp27NcMdhnGpmJNkXSZI0LYzp9FOPTwOHAa8Yp75MpozHTpYsWcLM\nmTO32rZo0SIWLVo0HruXJGlaGxgYYGBgYKttGzZsmLDj7VCoSXIJ8BrguFLKLzpN62gCQx9bz370\nAXd2avZJMqNntqavbRuq6V0NtSdwYE/NkT1d6+u0DX3tG6amjKJmYynlcUawbNky5s2bN1KJJEm7\nreH+o79q1Sr6+/sn5HhjPv3UBpq/orlAd223rZRyP01IWNCpn0FzHcz32k13AE/11MwBDgZuazfd\nBhzQXtQ7ZAFNYLq9U3N4klmdmoXABuCeTs38NhB1a9aUUjZ0ahawtYWdvkiSpGlgrPep+TTwJuA0\nYFOSvvaxX6fsYuDDSf4iyeHAl4AHga/Bby8cvhy4qL0/TD9wBXBrKWVlW3MvzcW6n09yZJJX0Cwl\nH2hXPgHcRBNermzvRXMicD5wSSnlybbmauAJ4IokhyU5FXgf8IlOfz8LvCjJBUnmJDkDeD1w0VjG\nRpIkTa2xnn56F82pm2/3bH8rTXihlHJhkmfQ3FPmAOCfgZNKKU906pcATwPXAvsCNwLv6dnnacAl\nNCuTtrS1Zw01llK2JDkF+AzNLNAm4IvAuZ2ajUkWApcCPwAGgaWllMs7NQ8kORlYRhN4HqRZAt67\nIkqSJO3CxnqfmlHN7JRSlgJLR2h/HDizfWyr5lc095kZ6Tg/A07ZTs3dwPHbqVlBc68dSZI0TfnZ\nT5IkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJU\nBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaS\nJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCo\nkSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQq\nGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhX2muoO\nSNq1rV69eqq7MCFmzZrFwQcfPNXdkDSODDWStuEXwB4sXrx4qjsyIfbb7xmsWbPaYCNVxFAjaRt+\nBWwBrgLmTnFfxttqNm9ezODgoKFGqsiYQ02S44CzgX7gucBrSylf77R/ATi959tuLKW8plOzL3AR\ncCqwL7AcOKOU8nCn5lnAJcApNO+sXwHOKqVs6tQcBHwWOAH4NfAl4G9KKVs6NUe0+zkSeBi4pJTy\n8Z7XdALwCeAlwFrgv5dS/mGMQyNVai4wb6o7IUnbtSMXCu8P/BA4AyjbqPkG0AfMbh+LetovBk4G\nXgfMB55HE1q6rqZ5N13Q1s4HPjfUmGQP4AaaYHYMTZB6C3Bep+aZNIHpfpp35bOBpUne0ak5BLgO\nuAV4KfBJ4LIkrx5xFCRJ0i5lzDM1pZQbgRsBkmQbZY+XUn45XEOSGcDbgDeWUr7TbnsrsDrJUaWU\nlUnmAicC/aWUO9uaM4Hrk3yglLKubX8x8KpSyiBwV5KPAB9LsrSU8hSwGNgbeHv7fHWSlwHvBy5r\nu/Ru4CellHPa52uSvBJYAnxzrOMjSZKmxkQt6T4hyfok9yb5dJIDO239NGHqlqENpZQ1NKd9jm03\nHQM8OhRoWjfTzAwd3am5qw00Q5YDM2lOIw3VrGgDTbdmTpKZnZqbe/q/vNMXSZI0DUxEqPkG8Gbg\nT4FzgOOBGzqzOrOBJ0opG3u+b33bNlTzcLexlPI08EhPzfph9sE41cxor/2RJEnTwLivfiqlXNN5\n+qMkdwE/prmY91vjfbwdtK3TZmOyZMkSZs6cudW2RYsWsWhR7yVEkiTtfgYGBhgYGNhq24YNGybs\neBO+pLuUcn+SQeBQmlCzDtgnyYye2Zq+to3263O6+0myJ3BgT82RPYfr67QNfe0bpqaMomZjKeXx\nkV7bsmXLmDfPVSGSJA1nuP/or1q1iv7+/gk53oR/TEKS5wPPprmTF8AdwFM0q5qGauYABwO3tZtu\nAw5oL+odsoBmhuX2Ts3hSWZ1ahYCG4B7OjXz20DUrVlTStnQqVnA1hZ2+iJJkqaBMYeaJPsneWmS\nP243vah9flDbdmGSo5O8IMkC4J+A+2guvqWdnbkcuCjJCUn6gSuAW0spK9uae9v6zyc5MskrgE8B\nA+3KJ4CbaMLLlUmOSHIicD7NfWiebGuuBp4ArkhyWJJTgffR3JNmyGfb13BBkjlJzgBeT3MfHUmS\nNE3syOmnl9OcRirtYygg/APNvWuOoLlQ+ADgIZpw8nedoAHNcumngWtpbr53I/CenuOcRnPTvJtp\nbr53LXDWUGMpZUuSU4DPAN8DNgFfBM7t1GxMshC4FPgBMAgsLaVc3ql5IMnJwDKawPMgzRLw3hVR\nkiRpF7Yj96n5DiPP8Pz5KPbxOHBm+9hWza9o7jMz0n5+RnPH4ZFq7qZZgTVSzQqapeaSJGmamvBr\naiRJkiaDoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIk\nVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiR\nJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoY\naiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSp\nCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqsNdUd0DDW7t2LYODg1PdDe1CVq9ePdVdkKRdmqFmF7R2\n7VrmzJnL5s2PTXVXJEmaNgw1u6DBwcE20FwFzJ3q7miXcQPwkanuhCTtsgw1u7S5wLyp7oR2GZ5+\nkqSReKGwJEmqgqFGkiRVwVAjSZKqYKiRJElVGHOoSXJckq8n+XmSLUn+cpia85I8lOSxJN9McmhP\n+75JLk0ymOTXSa5N8pyemmcl+XKSDUkeTXJZkv17ag5Kcn2STUnWJbkwyR49NUckWZHkN0l+muTs\nYfp7QpI7kmxOcl+S08c6LpIkaWrtyEzN/sAPgTOA0tuY5IPAe4G/Bo4CNgHLk+zTKbsYOBl4HTAf\neB7wlZ5dXU2z/GdBWzsf+FznOHvQrHHdCzgGOB14C3Bep+aZwHLgfpplRGcDS5O8o1NzCHAdcAvw\nUuCTwGVJXj3K8ZAkSbuAMS/pLqXcCNwIkCTDlJwFnF9Kua6teTOwHngtcE2SGcDbgDeWUr7T1rwV\nWJ3kqFLKyiRzgROB/lLKnW3NmcD1ST5QSlnXtr8YeFUpZRC4K8lHgI8lWVpKeQpYDOwNvL19vjrJ\ny4D3A5e1/X038JNSyjnt8zVJXgksAb451vGRJElTY1yvqUnyQmA2zawHAKWUjcDtwLHtppfThKlu\nzRpgbafmGODRoUDTuplmZujoTs1dbaAZshyYCbykU7OiDTTdmjlJZnZqbu55Kcs7fZEkSdPAeF8o\nPJsmeKzv2b6+bQPoA55ow862amYDD3cbSylPA4/01Ax3HMapZkaSfZEkSdPC7npH4eFOm43ZkiVL\nmDlz5lbbFi1axKJFi8Zj95IkTWsDAwMMDAxstW3Dhg0TdrzxDjXraAJDH1vPfvQBd3Zq9kkyo2e2\npq9tG6rpXQ21J3BgT82RPcfv67QNfe0bpqaMomZjKeXxf/8Sf2fZsmXMm+fHGEiSNJzh/qO/atUq\n+vv7J+R443r6qZRyP01IWDC0rb0w+Gjge+2mO4CnemrmAAcDt7WbbgMOaC/qHbKAJjDd3qk5PMms\nTs1CYANwT6dmfhuIujVrSikbOjUL2NrCTl8kSdI0sCP3qdk/yUuT/HG76UXt84Pa5xcDH07yF0kO\nB74EPAh8DX574fDlwEXt/WH6gSuAW0spK9uae2ku1v18kiOTvAL4FDDQrnwCuIkmvFzZ3ovmROB8\n4JJSypNtzdXAE8AVSQ5LcirwPuATnZf02fY1XJBkTpIzgNcDF411bCRJ0tTZkdNPLwe+RXMKp/C7\ngPAPwNtKKRcmeQbNPWUOAP4ZOKmU8kRnH0uAp4FrgX1ploi/p+c4pwGX0KxM2tLWnjXUWErZkuQU\n4DM0s0CbgC8C53ZqNiZZCFwK/AAYBJaWUi7v1DyQ5GRgGU3geZBmCXjviihJkrQL25H71HyH7czw\nlFKWAktHaH8cOLN9bKvmVzT3mRnpOD8DTtlOzd3A8dupWQFMzAk+SZI0KfzsJ0mSVAVDjSRJqoKh\nRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKq\nYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiS\npCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1\nkiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQF\nQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIk\nVWGv8d5hknOBc3s231tKOaxTcx7wDuAA4Fbg3aWUf+207wtcBJwK7AssB84opTzcqXkWcAlwCrAF\n+ApwVillU6fmIOCzwAnAr4EvAX9TStnSqTmi3c+RwMPAJaWUj+/cKEiaDlavXj3VXZCGNWvWLA4+\n+OCp7sa0M+6hpnU3sABI+/ypoYYkHwTeC7wZeAD4b8DyJHNLKU+0ZRcDJwGvAzYCl9KEluM6x7ga\n6GuPsw/wReBzwOL2OHsANwAPAccAzwOuBJ4APtzWPJMmMN0EvBM4HPhCkkdLKZeNx0BI2hX9AtiD\nxYsXT3VHpGHtt98zWLNmtcFmjCYq1DxVSvnlNtrOAs4vpVwHkOTNwHrgtcA1SWYAbwPeWEr5Tlvz\nVmB1kqNKKSuTzAVOBPpLKXe2NWcC1yf5QCllXdv+YuBVpZRB4K4kHwE+lmRpKeUpmgC0N/D29vnq\nJC8D3g8YaqRq/YpmgvcqYO4U90XqtZrNmxczODhoqBmjiQo1f5jk58Bm4DbgQ6WUnyV5ITAbuGWo\nsJSyMcntwLHANcDL2351a9YkWdvWrKSZeXl0KNC0bgYKcDTwtbbmrjbQDFkOfAZ4CfAvbc2KNtB0\na85JMrOUsmHnh0LSrmsuMG+qOyFpnEzEhcLfB95CM1PyLuCFwIok+9MEmkIzM9O1vm2D5pTSE6WU\njSPUzKa5/uW3SilPA4/01Ax3HMZYI0mSpoFxn6kppSzvPL07yUrgp8AbgHvH+3hTacmSJcycOXOr\nbYsWLWLRokVT1CNJknYdAwMDDAwMbLVtw4aJOwkyUaeffquUsiHJfcChwLdpLh7uY+sZkj5g6FTS\nOmCfJDN6Zmv62rahmud0j5NkT+DAnpoje7rT12kb+tq3nZptWrZsGfPmOXUtSdJwhvuP/qpVq+jv\n75+Q4034fWqS/D5NoHmolHI/TVhY0GmfQXMdzPfaTXfQrJbq1swBDqa5Pof26wHtRb1DhlZb3d6p\nOTzJrE7NQmADcE+nZn4biLo1a7yeRpKk6WXcQ02SjyeZn+QFSf4E+CrwJPCPbcnFwIeT/EWSw2nu\nHfMgzcW9tLMzlwMXJTkhST9wBXBrKWVlW3MvzQW9n09yZJJXAJ8CBtqVT9As074HuDLJEUlOBM6n\nuQ/Nk23N1TRLvK9IcliSU4H3AZ8Y73GRJEkTayJOPz2fJiw8G/gl8F3gmFLKvwGUUi5M8gyae8oc\nAPwzcFLnHjUAS4CngWtpbr53I/CenuOcRnPTvJtp1mZeS7NcnPY4W5KcQrPa6XvAJpp72ZzbqdmY\nZCHNfXB+AAwCS0spl+/0KEiSpEk1ERcKb/cq2VLKUmDpCO2PA2e2j23V/Ir2Rnsj1PyM5o7DI9Xc\nDRw/Uo0kSdr1+dlPkiSpCoYaSZJUBUONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIV\nDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpgqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmS\nVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKkKhhqJElSFQw1kiSpCoYaSZJUBUONJEmqgqFG\nkiRVwVAjSZKqYKiRJElVMNRIkqQqGGokSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXBUCNJkqpg\nqJEkSVUw1EiSpCoYaiRJUhUMNZIkqQqGGkmSVAVDjSRJqoKhRpIkVcFQI0mSqmCokSRJVTDUSJKk\nKhhqJElSFQw1kiSpCoYaSZJUBUONppGBqe6ANAn8PZ98jnktDDWtJO9Jcn+S3yT5fpIjp7pP6uUb\nj3YH/p5PPse8FoYaIMmpwCeAc4GXAf8CLE8ya0o7JkmSRs1Q01gCfK6U8qVSyr3Au4DHgLdNbbck\nSdJo7fahJsneQD9wy9C2UkoBbgaOnap+SZKksdlrqjuwC5gF7Ams79m+Hpizje/ZD2D16tUT0qHf\n7fcGYGKOMT09CHx5qjsxhW5tv07W78VkH28y7cqvbXf/PZ8Ku9qY3w9M3N+YqdZ5XfuN977TTErs\nvpI8F/g5cGwp5fbO9guA+aWUfzdbk+Q0dq1/AZIkTTdvKqVcPZ47dKYGBoGngb6e7X3Aum18z3Lg\nTcADwOYJ65kkSfXZDziE5m/puNrtZ2oAknwfuL2Uclb7PMBa4O9LKR+f0s5JkqRRcaamcRHwxSR3\nACtpVkM9A/jiVHZKkiSNnqEGKKVc096T5jya004/BE4spfxyansmSZJGy9NPkiSpCrv9fWokSVId\nDDWSJKkKhpoRJPnbJLcm2ZTkkW3UHJTk+rZmXZILk+zRU3NEkhXth2X+NMnZk/MKpr8kDyTZ0nk8\nneScnprt/gw0Nn7A68RJcm7P7/SWJPf01JyX5KEkjyX5ZpJDp6q/01GS45J8PcnP2/H9y2FqRhzj\nJPsmuTTJYJJfJ7k2yXMm71VML9sb8yRfGOb3/oaemp0ec9/4R7Y3cA3wmeEa2z+cN9BccH0McDrw\nFpoLjodqnkmzFv9+YB5wNrA0yTsmsuMVKcCHaS7gng08F/jUUONofgYaGz/gdVLcze9+p2cDrxxq\nSPJB4L3AXwNHAZtoxn+fKejndLU/zYKPM2jeQ7YyyjG+GDgZeB0wH3ge8JWJ7fa0NuKYt77B1r/3\ni3rad37MSyk+tvOg+UP5yDDbTwKeBGZ1tr0TeBTYq33+bpob/O3VqfkocM9Uv67p8KAJg+8boX27\nPwMfYx7z7wOf7DwPzX3kz5nqvtXwoAmLq0ZofwhY0nk+A/gN8Iap7vt0fABbgL8cyxi3zx8H/nOn\nZk67r6Om+jXt6o9tjPkXgP81wveMy5g7U7NzjgHuKqUMdrYtB2YCL+nUrCilPNVTMyfJzMnp5rT3\nN+105KokH0iyZ6dtND8DjZIf8Dpp/rCdpv9xkquSHASQ5IU0/4Ptjv9G4HYc/3ExyjF+Oc3sb7dm\nDc1NWf057LgTkqxPcm+STyc5sNPWzziMuaFm58xm+A/CHGobbY227ZPAG4ETgM8Cfwtc0Gl3fMfX\nSB/w6niOj+/TnCI9EXgX8EJgRZL9aca44PhPpNGMcR/wRBt2tlWjsfkG8GbgT4FzgOOBG9o7+EMz\nrjs95rvdzfeSfBT44AglBZhbSrlvkrq02xnLz6CUcnFn+91JngA+l+RDpZQnJ7Sj0gQopXQ/7+bu\nJCuBnwLbOX+wAAACvElEQVRvAO6dml5JE6uUck3n6Y+S3AX8mOY/rN8ar+PsdqEG+B805/ZG8pNR\n7msd0LsqpK/TNvR1uA/L7NbsbnbmZ7CS5vf2EOD/MrqfgUZvRz7gVTuhlLIhyX3AocC3aa5h6mPr\nmYQ+4M7J712V1rH9MV4H7JNkRs/Mgf8Oxkkp5f4kgzS/999inMZ8tzv9VEr5t3YGYKTHU9vfEwC3\nAYf3rApZCGwA7unUzO+5DmQhsKaUsmGnX9A0tJM/g5fRXDj2cPt8ND8DjVI7+3UHsGBoWzs9vAD4\n3lT1q2ZJfp/mjf2hUsr9NG/g3fGfARyN4z8uRjnGdwBP9dTMAQ6mec/RTkryfODZwC/aTeMy5rvj\nTM2otRfvHQi8ANgzyUvbpn8tpWwCbqL5w3llu0TwucD5wCWdUyNXA38HXJHkAuBw4H3AWZP3Sqan\nJMfQvNF8C/g18Cc0Hz56ZScQjuZnoLHxA14nUJKPA/+b5pTTfwD+K80Kvn9sSy4GPpzkX4EHaH6f\nHwS+Numdnaba65MOpZmRAXhR+/79SCnlZ2xnjEspG5NcDlyU5FGa95+/B24tpayc1BczTYw05u3j\nXJrl2evauguA+2gWdozfmE/10q9d+UFziuTpYR7zOzUHAdcB/49mKvMCYI+e/fwn4DvAYzRXcn9g\nql/bdHjQzMrc1v6D2ERzb49zgL176rb7M/Ax5rE/g+bN/jftz+DlU92nWh7AAM0f0N+07wdXAy/s\nqVlKs+z4sfZN/9Cp7vd0etBchLplmPfuK0Y7xsC+NPfEGmz/wP5P4DlT/dp21cdIYw7sB9xIE2g2\n01xe8BngD8Z7zP1AS0mSVIXd7poaSZJUJ0ONJEmqgqFGkiRVwVAjSZKqYKiRJElVMNRIkqQqGGok\nSVIVDDWSJKkKhhpJklQFQ40kSaqCoUaSJFXh/wN4E3OwJK3Z9AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x110c13d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "buckets = [-87.0, -15, 0, 30, 120]\n",
    "rdd_histogram_data = ml_bucketized_features\\\n",
    "  .select(\"ArrDelay\")\\\n",
    "  .rdd\\\n",
    "  .flatMap(lambda x: x)\\\n",
    "  .histogram(buckets)\n",
    "\n",
    "create_hist(rdd_histogram_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These look about right! The buckets `[-87, -15, 0, 30, 120]` correspond to human interprettable categories: `[early, on-time, late, very late]`. Lets use these buckets to recompute `ArrDelayBucket` and see if our problem goes away!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Recomputing Partitions for `ArrDelayBucket`\n",
    "\n",
    "Now that we have partitions that seem balanced, we can apply them to our features `DataFrame`. Lets recompute the Bucketizer and save it for use below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pyspark.ml.feature import Bucketizer\n",
    "\n",
    "# Setup the Bucketizer\n",
    "splits = [-float(\"inf\"), -15.0, 0, 30.0, float(\"inf\")]\n",
    "arrival_bucketizer = Bucketizer(\n",
    "  splits=splits,\n",
    "  inputCol=\"ArrDelay\",\n",
    "  outputCol=\"ArrDelayBucket\"\n",
    ")\n",
    "\n",
    "# Save the model\n",
    "arrival_bucketizer_path = \"{}/models/arrival_bucketizer_2.0.bin\".format(base_path)\n",
    "arrival_bucketizer.write().overwrite().save(arrival_bucketizer_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Inspecting the new `ArrDelayBucket`\n",
    "\n",
    "Now lets apply the new buckets to our data, take a sample and take a peek..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------+\n",
      "|ArrDelay|ArrDelayBucket|\n",
      "+--------+--------------+\n",
      "|     8.0|           2.0|\n",
      "|    49.0|           3.0|\n",
      "|   151.0|           3.0|\n",
      "|   -11.0|           1.0|\n",
      "|    28.0|           2.0|\n",
      "|     0.0|           2.0|\n",
      "|    -6.0|           1.0|\n",
      "|    -1.0|           1.0|\n",
      "|   -18.0|           0.0|\n",
      "|   -27.0|           0.0|\n",
      "|    -8.0|           1.0|\n",
      "|     1.0|           2.0|\n",
      "+--------+--------------+\n",
      "only showing top 12 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Apply it to our data\n",
    "ml_bucketized_features = arrival_bucketizer.transform(features)\n",
    "\n",
    "ml_bucketized_features\\\n",
    "  .select(\"ArrDelay\", \"ArrDelayBucket\")\\\n",
    "  .sample(False, 0.01, 27)\\\n",
    "  .show(12)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We're seeing examples of each label in our data. This is good. This looks differentiable. Now lets load all our models and see what happens..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Our Feature Models\n",
    "Lets load all the models we've saved that vecotirize our raw data, using code from [`ch08/make_predictions.py`](https://github.com/rjurney/Agile_Data_Code_2/blob/master/ch08/make_predictions.py)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Load the arrival delay bucketizer\n",
    "from pyspark.ml.feature import Bucketizer\n",
    "arrival_bucketizer_path = \"{}/models/arrival_bucketizer_2.0.bin\".format(base_path)\n",
    "arrival_bucketizer = Bucketizer.load(arrival_bucketizer_path)\n",
    "\n",
    "# Load the departure delay bucketizer\n",
    "departure_bucketizer_path = \"{}/models/departure_bucketizer.bin\".format(base_path)\n",
    "departure_bucketizer = Bucketizer.load(departure_bucketizer_path)\n",
    "\n",
    "# Load all the string field vectorizer pipelines into a dict\n",
    "from pyspark.ml import PipelineModel\n",
    "\n",
    "string_vectorizer_pipeline_models = {}\n",
    "for column in [\"Carrier\", \"DayOfMonth\", \"DayOfWeek\", \"DayOfYear\",\n",
    "               \"Origin\", \"Dest\", \"FlightNum\", \"DepDelayBucket\"]:\n",
    "  string_pipeline_model_path = \"{}/models/string_indexer_pipeline_model_{}.bin\".format(\n",
    "    base_path,\n",
    "    column\n",
    "  )\n",
    "  string_pipeline_model = PipelineModel.load(string_pipeline_model_path)\n",
    "  string_vectorizer_pipeline_models[column] = string_pipeline_model\n",
    "\n",
    "# Load the numeric vector assembler\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "vector_assembler_path = \"{}/models/numeric_vector_assembler.bin\".format(base_path)\n",
    "vector_assembler = VectorAssembler.load(vector_assembler_path)\n",
    "\n",
    "# Load the final assembler\n",
    "final_assembler_path = \"{}/models/final_vector_assembler.bin\".format(base_path)\n",
    "final_assembler = VectorAssembler.load(final_assembler_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorize our Training Data\n",
    "Having loaded all the models that vectorize our data, lets use them to vectorize our data! This will prepare it for re-training our classifier model in the next step."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------+\n",
      "|ArrDelay|ArrDelayBucket|\n",
      "+--------+--------------+\n",
      "|    13.0|           2.0|\n",
      "|    17.0|           2.0|\n",
      "|    36.0|           3.0|\n",
      "|   -21.0|           0.0|\n",
      "|   -14.0|           1.0|\n",
      "|    16.0|           2.0|\n",
      "|    -7.0|           1.0|\n",
      "|    13.0|           2.0|\n",
      "|    25.0|           2.0|\n",
      "|    58.0|           3.0|\n",
      "|    14.0|           2.0|\n",
      "|     1.0|           2.0|\n",
      "+--------+--------------+\n",
      "only showing top 12 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Bucketize the departure and arrival delays for classification\n",
    "ml_bucketized_features = arrival_bucketizer.transform(features)\n",
    "\n",
    "# Check the buckets\n",
    "ml_bucketized_features.select(\"ArrDelay\", \"ArrDelayBucket\").show(12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Vectorize string fields with the corresponding pipeline for that column\n",
    "# Turn category fields into categoric feature vectors, then drop intermediate fields\n",
    "for column in [\"Carrier\", \"DayOfMonth\", \"DayOfWeek\", \"DayOfYear\",\n",
    "               \"Origin\", \"Dest\", \"FlightNum\"]:\n",
    "  string_pipeline_path = \"{}/models/string_indexer_pipeline_{}.bin\".format(\n",
    "    base_path,\n",
    "    column\n",
    "  )\n",
    "  string_pipeline_model = string_vectorizer_pipeline_models[column]\n",
    "  ml_bucketized_features = string_pipeline_model.transform(ml_bucketized_features)\n",
    "  ml_bucketized_features = ml_bucketized_features.drop(column + \"_index\")\n",
    "\n",
    "# Vectorize numeric columns: DepDelay and Distance\n",
    "ml_bucketized_features = vector_assembler.transform(ml_bucketized_features)\n",
    "\n",
    "# Combine various features into one feature vector, 'features'\n",
    "final_vectorized_features = final_assembler.transform(ml_bucketized_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cleanup by dropping the individual vectors and check the data out..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|        Features_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|(8009,[2,39,45,37...|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|(8009,[2,39,45,37...|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|(8009,[2,39,45,37...|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|(8009,[2,39,45,37...|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|(8009,[2,39,45,37...|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Drop the individual vector columns\n",
    "feature_columns = [\"Carrier_vec\", \"DayOfMonth_vec\", \"DayOfWeek_vec\", \"DayOfYear_vec\",\n",
    "                   \"Origin_vec\", \"Dest_vec\", \"FlightNum_vec\", \"NumericFeatures_vec\"]\n",
    "for column in feature_columns:\n",
    "  final_vectorized_features = final_vectorized_features.drop(column)\n",
    "\n",
    "final_vectorized_features.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retraining Our Classifier Model\n",
    "And now we can retrain our classifier using the new `ArrDelayBuckets`, save it for use elsewhere and then test the new model and see if it still has the same problem."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Run: 100K\n",
    "Lets start out with just 100K training items to see that it runs ok... this way we don't wait a long time just to find out the model fails to fit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|        Features_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|(8009,[2,39,45,37...|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|(8009,[2,39,45,37...|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|(8009,[2,39,45,37...|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|(8009,[2,39,45,37...|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|(8009,[2,39,45,37...|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n",
      "Model trained on 100K items successfully!\n"
     ]
    }
   ],
   "source": [
    "# Inspect the finalized features - try first with just 100K items\n",
    "limited_final_vectorized_features = final_vectorized_features.limit(100000)\n",
    "limited_final_vectorized_features.show(5)\n",
    "\n",
    "# Instantiate and fit random forest classifier on all the data\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "rfc = RandomForestClassifier(featuresCol=\"Features_vec\", labelCol=\"ArrDelayBucket\", predictionCol=\"Prediction\")\n",
    "model = rfc.fit(limited_final_vectorized_features)\n",
    "\n",
    "print(\"Model trained on 100K items successfully!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Run: 5.7MM\n",
    "The model having successfully fitted on limited data, lets train it on all 1.5 million flights and save the result to our `models/` directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model trained on 5.7MM items successfully!\n"
     ]
    }
   ],
   "source": [
    "# Instantiate and fit random forest classifier on all the data\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "rfc = RandomForestClassifier(featuresCol=\"Features_vec\", labelCol=\"ArrDelayBucket\", predictionCol=\"Prediction\")\n",
    "#model = rfc.fit(final_vectorized_features)\n",
    "model = rfc.fit(limited_final_vectorized_features)\n",
    "\n",
    "# Save the new model over the old one\n",
    "model_output_path = \"{}/models/spark_random_forest_classifier.flight_delays.2.0.bin\".format(\n",
    "  base_path\n",
    ")\n",
    "model.write().overwrite().save(model_output_path)\n",
    "\n",
    "print(\"Model trained on 5.7MM items successfully!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing Our New Classifier Model\n",
    "Now we need to test and see whether the new model makes predictions other than 0.0 :) Lets make a prediction for our training data and count the different labels it predicts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-------+\n",
      "|Prediction|  count|\n",
      "+----------+-------+\n",
      "|       1.0|  32887|\n",
      "|       2.0|5681121|\n",
      "+----------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make the predictions...\n",
    "predictions = model.transform(final_vectorized_features)\n",
    "predictions.groupBy(\"Prediction\").count().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Shit! Ok, the bucketing wasn't the issue. DAMN. Well, this is a learning experience. We'll have to figure out something else."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### In Which We Regroup And Figure Things Out\n",
    "This seems more like a bug. What the hell is going on?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Take 2: Simplify The Model\n",
    "Lets pull all but one feature out and see what happens.\n",
    "\n",
    "### Load the Data Again"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Row(ArrDelay=13.0, CRSArrTime=datetime.datetime(2015, 1, 1, 10, 10), CRSDepTime=datetime.datetime(2015, 1, 1, 7, 30), Carrier='AA', DayOfMonth=1, DayOfWeek=4, DayOfYear=1, DepDelay=14.0, Dest='DFW', Distance=569.0, FlightDate=datetime.date(2014, 12, 31), FlightNum='1024', Origin='ABQ')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "schema = StructType([\n",
    "  StructField(\"ArrDelay\", DoubleType(), True),     # \"ArrDelay\":5.0\n",
    "  StructField(\"CRSArrTime\", TimestampType(), True),    # \"CRSArrTime\":\"2015-12-31T03:20:00.000-08:00\"\n",
    "  StructField(\"CRSDepTime\", TimestampType(), True),    # \"CRSDepTime\":\"2015-12-31T03:05:00.000-08:00\"\n",
    "  StructField(\"Carrier\", StringType(), True),     # \"Carrier\":\"WN\"\n",
    "  StructField(\"DayOfMonth\", IntegerType(), True), # \"DayOfMonth\":31\n",
    "  StructField(\"DayOfWeek\", IntegerType(), True),  # \"DayOfWeek\":4\n",
    "  StructField(\"DayOfYear\", IntegerType(), True),  # \"DayOfYear\":365\n",
    "  StructField(\"DepDelay\", DoubleType(), True),     # \"DepDelay\":14.0\n",
    "  StructField(\"Dest\", StringType(), True),        # \"Dest\":\"SAN\"\n",
    "  StructField(\"Distance\", DoubleType(), True),     # \"Distance\":368.0\n",
    "  StructField(\"FlightDate\", DateType(), True),    # \"FlightDate\":\"2015-12-30T16:00:00.000-08:00\"\n",
    "  StructField(\"FlightNum\", StringType(), True),   # \"FlightNum\":\"6109\"\n",
    "  StructField(\"Origin\", StringType(), True),      # \"Origin\":\"TUS\"\n",
    "])\n",
    "\n",
    "input_path = \"{}/data/simple_flight_delay_features.json\".format(\n",
    "  base_path\n",
    ")\n",
    "features = spark.read.json(input_path, schema=schema)\n",
    "features.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bucketize Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Bucketize the departure and arrival delays for classification\n",
    "ml_bucketized_features = arrival_bucketizer.transform(features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Only Vectorize the Numeric Columns\n",
    "`DepDelay` and `Distance` will be the entirety of our model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Vectorize numeric columns: DepDelay and Distance\n",
    "numeric_features = vector_assembler.transform(ml_bucketized_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a New Random Forest Classifier Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model trained on 5.7MM items successfully!\n"
     ]
    }
   ],
   "source": [
    "# Instantiate and fit random forest classifier on all the data\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "rfc = RandomForestClassifier(featuresCol=\"NumericFeatures_vec\", labelCol=\"ArrDelayBucket\", predictionCol=\"Prediction\")\n",
    "model = rfc.fit(numeric_features)\n",
    "\n",
    "# Save the new model over the old one\n",
    "model_output_path = \"{}/models/spark_random_forest_classifier.flight_delays.3.0.bin\".format(\n",
    "  base_path\n",
    ")\n",
    "model.write().overwrite().save(model_output_path)\n",
    "\n",
    "print(\"Model trained on 5.7MM items successfully!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make the Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------------------+--------------------+--------------------+----------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|NumericFeatures_vec|       rawPrediction|         probability|Prediction|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------------------+--------------------+--------------------+----------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|       [14.0,569.0]|[0.04395894123984...|[0.00219794706199...|       2.0|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|       [14.0,569.0]|[0.04395894123984...|[0.00219794706199...|       2.0|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|       [-2.0,569.0]|[2.25610771946355...|[0.11280538597317...|       1.0|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|       [-1.0,731.0]|[5.44416799386120...|[0.27220839969306...|       1.0|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|       [-4.0,731.0]|[7.51562274287558...|[0.37578113714377...|       1.0|\n",
      "|    16.0|2015-01-01 07:15:...|2015-01-01 05:45:...|     AA|         1|        4|        1|    15.0| DFW|   731.0|2014-12-31|     1473|   ATL|           2.0|       [15.0,731.0]|[0.04395894123984...|[0.00219794706199...|       2.0|\n",
      "|    -7.0|2015-01-01 04:15:...|2015-01-01 02:45:...|     AA|         1|        4|        1|    -2.0| DFW|   731.0|2014-12-31|     1513|   ATL|           1.0|       [-2.0,731.0]|[5.44416799386120...|[0.27220839969306...|       1.0|\n",
      "|    13.0|2015-01-01 08:50:...|2015-01-01 07:25:...|     AA|         1|        4|        1|     9.0| DFW|   731.0|2014-12-31|      194|   ATL|           2.0|        [9.0,731.0]|[1.24591307802481...|[0.06229565390124...|       2.0|\n",
      "|    25.0|2015-01-01 12:30:...|2015-01-01 11:00:...|     AA|         1|        4|        1|    -2.0| DFW|   731.0|2014-12-31|      232|   ATL|           2.0|       [-2.0,731.0]|[5.44416799386120...|[0.27220839969306...|       1.0|\n",
      "|    58.0|2015-01-01 13:40:...|2015-01-01 12:15:...|     AA|         1|        4|        1|    14.0| DFW|   731.0|2014-12-31|      276|   ATL|           3.0|       [14.0,731.0]|[0.04395894123984...|[0.00219794706199...|       2.0|\n",
      "|    14.0|2015-01-01 05:25:...|2015-01-01 03:55:...|     AA|         1|        4|        1|    15.0| DFW|   731.0|2014-12-31|      314|   ATL|           2.0|       [15.0,731.0]|[0.04395894123984...|[0.00219794706199...|       2.0|\n",
      "|     1.0|2015-01-01 10:05:...|2015-01-01 08:40:...|     AA|         1|        4|        1|    -5.0| DFW|   731.0|2014-12-31|      356|   ATL|           2.0|       [-5.0,731.0]|[7.51562274287558...|[0.37578113714377...|       1.0|\n",
      "|   -29.0|2015-01-01 02:12:...|2015-01-01 00:15:...|     AA|         1|        4|        1|    -9.0| MIA|   594.0|2014-12-31|     1652|   ATL|           0.0|       [-9.0,594.0]|[8.19704519542880...|[0.40985225977144...|       1.0|\n",
      "|   -10.0|2015-01-01 00:52:...|2014-12-31 23:00:...|     AA|         1|        4|        1|    -4.0| MIA|   594.0|2014-12-31|       17|   ATL|           1.0|       [-4.0,594.0]|[4.45234793730754...|[0.22261739686537...|       1.0|\n",
      "|    -3.0|2015-01-01 15:02:...|2015-01-01 13:10:...|     AA|         1|        4|        1|    -7.0| MIA|   594.0|2014-12-31|      349|   ATL|           1.0|       [-7.0,594.0]|[4.45234793730754...|[0.22261739686537...|       1.0|\n",
      "|    -8.0|2015-01-01 06:35:...|2015-01-01 05:30:...|     AA|         1|        4|        1|    -2.0| DFW|   190.0|2014-12-31|     1023|   AUS|           1.0|       [-2.0,190.0]|[2.25610771946355...|[0.11280538597317...|       1.0|\n",
      "|    -1.0|2014-12-31 22:50:...|2014-12-31 21:50:...|     AA|         1|        4|        1|    -2.0| DFW|   190.0|2014-12-31|     1178|   AUS|           1.0|       [-2.0,190.0]|[2.25610771946355...|[0.11280538597317...|       1.0|\n",
      "|   -14.0|2015-01-01 01:40:...|2015-01-01 00:30:...|     AA|         1|        4|        1|    -6.0| DFW|   190.0|2014-12-31|     1296|   AUS|           1.0|       [-6.0,190.0]|[4.45234793730754...|[0.22261739686537...|       1.0|\n",
      "|   -16.0|2015-01-01 02:15:...|2015-01-01 01:05:...|     AA|         1|        4|        1|    -4.0| DFW|   190.0|2014-12-31|     1356|   AUS|           0.0|       [-4.0,190.0]|[4.45234793730754...|[0.22261739686537...|       1.0|\n",
      "|    18.0|2015-01-01 08:55:...|2015-01-01 07:55:...|     AA|         1|        4|        1|     3.0| DFW|   190.0|2014-12-31|     1365|   AUS|           2.0|        [3.0,190.0]|[0.87508697792321...|[0.04375434889616...|       1.0|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------------------+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make the predictions...\n",
    "predictions = model.transform(numeric_features)\n",
    "predictions.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check the Distribution of Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------+\n",
      "|Prediction|count(1)|\n",
      "+----------+--------+\n",
      "|       0.0|  244210|\n",
      "|       1.0| 3916886|\n",
      "|       3.0|  542936|\n",
      "|       2.0| 1009976|\n",
      "+----------+--------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.registerTempTable(\"predictions\")\n",
    "spark.sql(\"\"\"SELECT Prediction, COUNT(*) FROM predictions GROUP BY Prediction\"\"\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare to Actual Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+--------+\n",
      "|ArrDelayBucket|count(1)|\n",
      "+--------------+--------+\n",
      "|           0.0| 1098212|\n",
      "|           1.0| 2402687|\n",
      "|           3.0|  650604|\n",
      "|           2.0| 1562505|\n",
      "+--------------+--------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark.sql(\"\"\"Select ArrDelayBucket, COUNT(*) FROM predictions GROUP BY ArrDelayBucket\"\"\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ok, this looks respectable, or at least... not completely erroneous! Now lets add in another feature and see what happens."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Adding Back String Columns\n",
    "Now, starting with just `Origin` and `Dest`, we'll try adding back the string columns and see what happens.\n",
    "\n",
    "Start by creating a vector for each field, `Origin` and `Dest`, and check them out. Note that we repeat the numeric feature vectorization to refresh the `numeric_features` relation, which we modify by looping."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------------------+----------------+---------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|NumericFeatures_vec|      Origin_vec|       Dest_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------------------+----------------+---------------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|       [14.0,569.0]|(322,[53],[1.0])|(322,[2],[1.0])|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|       [14.0,569.0]|(322,[53],[1.0])|(322,[2],[1.0])|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|       [-2.0,569.0]|(322,[53],[1.0])|(322,[2],[1.0])|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|       [-1.0,731.0]| (322,[0],[1.0])|(322,[2],[1.0])|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|       [-4.0,731.0]| (322,[0],[1.0])|(322,[2],[1.0])|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------------------+----------------+---------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Vectorize numeric columns: DepDelay and Distance\n",
    "numeric_features = vector_assembler.transform(ml_bucketized_features)\n",
    "\n",
    "# Vectorize string fields with the corresponding pipeline for that column\n",
    "# Turn category fields into categoric feature vectors, then drop intermediate fields\n",
    "string_columns = [\"Origin\", \"Dest\"]\n",
    "for column in string_columns:\n",
    "  string_pipeline_path = \"{}/models/string_indexer_pipeline_{}.bin\".format(\n",
    "    base_path,\n",
    "    column\n",
    "  )\n",
    "  string_pipeline_model = string_vectorizer_pipeline_models[column]\n",
    "  numeric_features = string_pipeline_model.transform(numeric_features)\n",
    "  numeric_features = numeric_features.drop(column + \"_index\")\n",
    "\n",
    "numeric_features.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assemble Our Vectors\n",
    "Now we need to assemble the `Origin`/`Dest`/`NumericFeatures` vectors into one `Features_vec`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|        Features_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|(646,[53,324,644,...|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|(646,[53,324,644,...|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|(646,[53,324,644,...|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|(646,[0,324,644,6...|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|(646,[0,324,644,6...|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "vector_columns = [\"Origin_vec\", \"Dest_vec\", \"NumericFeatures_vec\"]\n",
    "final_assembler = VectorAssembler(\n",
    "    inputCols=vector_columns,\n",
    "    outputCol=\"Features_vec\"\n",
    ")\n",
    "final_vectorized_features = final_assembler.transform(numeric_features)\n",
    "for column in vector_columns:\n",
    "  final_vectorized_features = final_vectorized_features.drop(column)\n",
    "\n",
    "final_vectorized_features.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrain the Model\n",
    "Note that as we add features, the model fitting slows down..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model trained on 5.7MM items successfully!\n"
     ]
    }
   ],
   "source": [
    "# Instantiate and fit random forest classifier on all the data\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "rfc = RandomForestClassifier(featuresCol=\"Features_vec\", labelCol=\"ArrDelayBucket\", predictionCol=\"Prediction\")\n",
    "model = rfc.fit(final_vectorized_features)\n",
    "\n",
    "# Save the new model over the old one\n",
    "model_output_path = \"{}/models/spark_random_forest_classifier.flight_delays.4.0.bin\".format(\n",
    "  base_path\n",
    ")\n",
    "model.write().overwrite().save(model_output_path)\n",
    "\n",
    "print(\"Model trained on 5.7MM items successfully!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Make our Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+--------------------+--------------------+----------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|        Features_vec|       rawPrediction|         probability|Prediction|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+--------------------+--------------------+----------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|(646,[53,324,644,...|[3.65207272881038...|[0.18260363644051...|       1.0|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|(646,[53,324,644,...|[3.65207272881038...|[0.18260363644051...|       1.0|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|(646,[53,324,644,...|[3.88802642603634...|[0.19440132130181...|       1.0|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|(646,[0,324,644,6...|[3.99362073652591...|[0.19968103682629...|       1.0|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|(646,[0,324,644,6...|[3.99362073652591...|[0.19968103682629...|       1.0|\n",
      "|    16.0|2015-01-01 07:15:...|2015-01-01 05:45:...|     AA|         1|        4|        1|    15.0| DFW|   731.0|2014-12-31|     1473|   ATL|           2.0|(646,[0,324,644,6...|[3.75766703929995...|[0.18788335196499...|       1.0|\n",
      "|    -7.0|2015-01-01 04:15:...|2015-01-01 02:45:...|     AA|         1|        4|        1|    -2.0| DFW|   731.0|2014-12-31|     1513|   ATL|           1.0|(646,[0,324,644,6...|[3.99362073652591...|[0.19968103682629...|       1.0|\n",
      "|    13.0|2015-01-01 08:50:...|2015-01-01 07:25:...|     AA|         1|        4|        1|     9.0| DFW|   731.0|2014-12-31|      194|   ATL|           2.0|(646,[0,324,644,6...|[3.75766703929995...|[0.18788335196499...|       1.0|\n",
      "|    25.0|2015-01-01 12:30:...|2015-01-01 11:00:...|     AA|         1|        4|        1|    -2.0| DFW|   731.0|2014-12-31|      232|   ATL|           2.0|(646,[0,324,644,6...|[3.99362073652591...|[0.19968103682629...|       1.0|\n",
      "|    58.0|2015-01-01 13:40:...|2015-01-01 12:15:...|     AA|         1|        4|        1|    14.0| DFW|   731.0|2014-12-31|      276|   ATL|           3.0|(646,[0,324,644,6...|[3.75766703929995...|[0.18788335196499...|       1.0|\n",
      "|    14.0|2015-01-01 05:25:...|2015-01-01 03:55:...|     AA|         1|        4|        1|    15.0| DFW|   731.0|2014-12-31|      314|   ATL|           2.0|(646,[0,324,644,6...|[3.75766703929995...|[0.18788335196499...|       1.0|\n",
      "|     1.0|2015-01-01 10:05:...|2015-01-01 08:40:...|     AA|         1|        4|        1|    -5.0| DFW|   731.0|2014-12-31|      356|   ATL|           2.0|(646,[0,324,644,6...|[3.99362073652591...|[0.19968103682629...|       1.0|\n",
      "|   -29.0|2015-01-01 02:12:...|2015-01-01 00:15:...|     AA|         1|        4|        1|    -9.0| MIA|   594.0|2014-12-31|     1652|   ATL|           0.0|(646,[0,346,644,6...|[3.79791319714944...|[0.18989565985747...|       1.0|\n",
      "|   -10.0|2015-01-01 00:52:...|2014-12-31 23:00:...|     AA|         1|        4|        1|    -4.0| MIA|   594.0|2014-12-31|       17|   ATL|           1.0|(646,[0,346,644,6...|[3.79791319714944...|[0.18989565985747...|       1.0|\n",
      "|    -3.0|2015-01-01 15:02:...|2015-01-01 13:10:...|     AA|         1|        4|        1|    -7.0| MIA|   594.0|2014-12-31|      349|   ATL|           1.0|(646,[0,346,644,6...|[3.79791319714944...|[0.18989565985747...|       1.0|\n",
      "|    -8.0|2015-01-01 06:35:...|2015-01-01 05:30:...|     AA|         1|        4|        1|    -2.0| DFW|   190.0|2014-12-31|     1023|   AUS|           1.0|(646,[34,324,644,...|[3.8828148227923,...|[0.19414074113961...|       1.0|\n",
      "|    -1.0|2014-12-31 22:50:...|2014-12-31 21:50:...|     AA|         1|        4|        1|    -2.0| DFW|   190.0|2014-12-31|     1178|   AUS|           1.0|(646,[34,324,644,...|[3.8828148227923,...|[0.19414074113961...|       1.0|\n",
      "|   -14.0|2015-01-01 01:40:...|2015-01-01 00:30:...|     AA|         1|        4|        1|    -6.0| DFW|   190.0|2014-12-31|     1296|   AUS|           1.0|(646,[34,324,644,...|[3.8828148227923,...|[0.19414074113961...|       1.0|\n",
      "|   -16.0|2015-01-01 02:15:...|2015-01-01 01:05:...|     AA|         1|        4|        1|    -4.0| DFW|   190.0|2014-12-31|     1356|   AUS|           0.0|(646,[34,324,644,...|[3.8828148227923,...|[0.19414074113961...|       1.0|\n",
      "|    18.0|2015-01-01 08:55:...|2015-01-01 07:55:...|     AA|         1|        4|        1|     3.0| DFW|   190.0|2014-12-31|     1365|   AUS|           2.0|(646,[34,324,644,...|[3.8828148227923,...|[0.19414074113961...|       1.0|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make the predictions...\n",
    "predictions = model.transform(final_vectorized_features)\n",
    "predictions.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Uh oh... all 1.0s..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check that the Results are Sane\n",
    "And drum roll... are the results sane?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+--------+\n",
      "|Prediction|count(1)|\n",
      "+----------+--------+\n",
      "|       1.0| 5699785|\n",
      "|       3.0|   14223|\n",
      "+----------+--------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.registerTempTable(\"predictions\")\n",
    "spark.sql(\"\"\"SELECT Prediction, COUNT(*) FROM predictions GROUP BY Prediction\"\"\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So... how can the performance of the model completely disappear when we add `Origin` and `Dest`? Even if these items have no contribution to the model... shouldn't it still compensate? It is a decision tree!\n",
    "\n",
    "### Classifier Model Feature Importances\n",
    "Lets look at the feature importances. What are they?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SparseVector(646, {0: 0.0303, 1: 0.061, 2: 0.0, 3: 0.0001, 4: 0.0087, 6: 0.0077, 7: 0.0048, 8: 0.0, 9: 0.0, 10: 0.0051, 13: 0.0006, 14: 0.0012, 15: 0.0657, 16: 0.001, 17: 0.0416, 18: 0.0001, 19: 0.0, 20: 0.0, 21: 0.0057, 22: 0.0048, 23: 0.0022, 25: 0.0006, 27: 0.011, 28: 0.0246, 29: 0.0004, 31: 0.0, 32: 0.0119, 33: 0.0138, 34: 0.0, 35: 0.0001, 36: 0.0003, 37: 0.013, 38: 0.0152, 39: 0.001, 40: 0.006, 41: 0.0009, 42: 0.0, 43: 0.0005, 46: 0.0, 47: 0.0004, 48: 0.0, 51: 0.0003, 53: 0.0002, 54: 0.0071, 56: 0.0002, 58: 0.0002, 61: 0.0, 66: 0.0, 67: 0.0002, 75: 0.0, 76: 0.0002, 79: 0.0, 84: 0.0, 86: 0.0025, 102: 0.0, 103: 0.0, 104: 0.0, 126: 0.0001, 127: 0.0, 137: 0.0, 138: 0.0, 139: 0.0001, 142: 0.0, 151: 0.0002, 155: 0.0004, 158: 0.0, 190: 0.0002, 195: 0.0001, 232: 0.0003, 301: 0.0, 322: 0.0202, 323: 0.0584, 324: 0.0072, 325: 0.0015, 326: 0.0009, 327: 0.0298, 328: 0.0036, 329: 0.0, 330: 0.0006, 331: 0.0004, 332: 0.0001, 334: 0.0007, 335: 0.0005, 336: 0.0006, 337: 0.0175, 338: 0.0048, 339: 0.0194, 340: 0.0507, 341: 0.0004, 342: 0.0, 343: 0.0005, 344: 0.0002, 345: 0.0002, 346: 0.0002, 347: 0.0005, 350: 0.0003, 351: 0.0001, 352: 0.0005, 354: 0.0094, 355: 0.0, 356: 0.0001, 357: 0.0001, 359: 0.0003, 361: 0.0004, 363: 0.0001, 365: 0.0001, 366: 0.0, 369: 0.0003, 373: 0.0003, 376: 0.0054, 378: 0.0002, 382: 0.0001, 388: 0.0, 389: 0.0002, 393: 0.0024, 395: 0.0, 396: 0.0034, 397: 0.0, 398: 0.0, 413: 0.0004, 417: 0.0003, 419: 0.0, 420: 0.0001, 428: 0.001, 436: 0.0002, 439: 0.0, 442: 0.0002, 444: 0.0, 448: 0.0, 453: 0.0008, 455: 0.0, 471: 0.0003, 472: 0.0002, 514: 0.0001, 524: 0.0002, 585: 0.0002, 644: 0.3348, 645: 0.0657})"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_importances = model.featureImportances\n",
    "feature_importances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Great, WTF Do I Do with This SparseVector?\n",
    "\n",
    "I have no idea what to do with this. This tells me nothing. How do I reverse the process to get the feature names?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# I dunno"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try Combining `Origin`/`Dest` Into One `Route` Column\n",
    "\n",
    "Try making it one feature, the route as defined by the `Origin`/`Dest` pair."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|  Route|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|ABQ-DFW|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|ABQ-DFW|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|ABQ-DFW|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|ATL-DFW|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|ATL-DFW|\n",
      "|    16.0|2015-01-01 07:15:...|2015-01-01 05:45:...|     AA|         1|        4|        1|    15.0| DFW|   731.0|2014-12-31|     1473|   ATL|           2.0|ATL-DFW|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+\n",
      "only showing top 6 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Bucketize the departure and arrival delays for classification\n",
    "ml_bucketized_features = arrival_bucketizer.transform(features)\n",
    "from pyspark.sql.functions import lit, concat\n",
    "features_with_route = ml_bucketized_features.withColumn(\n",
    "  'Route',\n",
    "  concat(\n",
    "    ml_bucketized_features.Origin,\n",
    "    lit('-'),\n",
    "    ml_bucketized_features.Dest\n",
    "  )\n",
    ")\n",
    "features_with_route.show(6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorize `Route`\n",
    "\n",
    "Now we must create a string indexer pipeline for the `Route` column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved route string pipeline model to ../models/string_indexer_pipeline_model_Route.bin...\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import StringIndexer, OneHotEncoder, VectorIndexer\n",
    "\n",
    "route_string_indexer = StringIndexer(\n",
    "  inputCol='Route',\n",
    "  outputCol='Route' + \"_index\"\n",
    ")\n",
    "\n",
    "route_one_hot_encoder = OneHotEncoder(\n",
    "  dropLast=False,\n",
    "  inputCol='Route' + \"_index\",\n",
    "  outputCol='Route' + \"_vec\"\n",
    ")\n",
    "route_string_pipeline = Pipeline(\n",
    "  stages=[route_string_indexer, route_one_hot_encoder]\n",
    ")\n",
    "\n",
    "# Fit/transform to apply to the data\n",
    "route_string_pipeline_model = route_string_pipeline.fit(features_with_route)\n",
    "features_with_route_vec = route_string_pipeline_model.transform(features_with_route)\n",
    "\n",
    "# Drop the intermediate index\n",
    "features_with_route_vec = features_with_route_vec.drop(\"Route\" + \"_index\")\n",
    "\n",
    "# Save the pipeline model\n",
    "route_string_pipeline_output_path = \"{}/models/string_indexer_pipeline_model_{}.bin\".format(\n",
    "  base_path,\n",
    "  \"Route\"\n",
    ")\n",
    "route_string_pipeline_model.write().overwrite().save(route_string_pipeline_output_path)\n",
    "\n",
    "print(\"Saved route string pipeline model to {}...\".format(route_string_pipeline_output_path))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Verify `Route_vec`\n",
    "And lets check the resulting `Route_vec` column..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-------------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|  Route|          Route_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-------------------+\n",
      "|    38.0|2015-01-01 08:50:...|2015-01-01 03:45:...|     AA|         1|        4|        1|    29.0| DFW|  1235.0|2014-12-31|     2419|   LAX|           3.0|LAX-DFW|  (4657,[45],[1.0])|\n",
      "|    53.0|2015-01-01 14:04:...|2015-01-01 12:55:...|     B6|         1|        4|        1|    62.0| BOS|   187.0|2014-12-31|      418|   JFK|           3.0|JFK-BOS| (4657,[129],[1.0])|\n",
      "|     1.0|2015-01-01 13:05:...|2015-01-01 09:54:...|     UA|         1|        4|        1|     3.0| MCO|   854.0|2014-12-31|     1586|   IAH|           2.0|IAH-MCO| (4657,[494],[1.0])|\n",
      "|   -17.0|2015-01-01 09:13:...|2015-01-01 04:55:...|     UA|         1|        4|        1|     0.0| PHL|  1325.0|2014-12-31|      234|   IAH|           0.0|IAH-PHL| (4657,[673],[1.0])|\n",
      "|   -19.0|2015-01-01 07:55:...|2015-01-01 03:30:...|     WN|         1|        4|        1|    -5.0| EWR|  1504.0|2014-12-31|     1760|   AUS|           0.0|AUS-EWR|(4657,[1768],[1.0])|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Show a sample of the output\n",
    "features_with_route_vec.sample(False, 0.001, 11).show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note that the thing exploded in size, up to 4657 routes.\n",
    "\n",
    "### Recreate `NumericFeatures_vec`\n",
    "We need to recreate the vectors for our numeric columns, `DepDelay` and `Distance`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+------------------+-------------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|  Route|         Route_vec|NumericFeatures_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+------------------+-------------------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|ABQ-DFW|(4657,[938],[1.0])|       [14.0,569.0]|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|ABQ-DFW|(4657,[938],[1.0])|       [14.0,569.0]|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|ABQ-DFW|(4657,[938],[1.0])|       [-2.0,569.0]|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|ATL-DFW| (4657,[37],[1.0])|       [-1.0,731.0]|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|ATL-DFW| (4657,[37],[1.0])|       [-4.0,731.0]|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+------------------+-------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Load the numeric vector assembler\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "vector_assembler_path = \"{}/models/numeric_vector_assembler.bin\".format(base_path)\n",
    "vector_assembler = VectorAssembler.load(vector_assembler_path)\n",
    "\n",
    "# Vectorize numeric columns: DepDelay and Distance\n",
    "features_with_route_and_numeric = vector_assembler.transform(features_with_route_vec)\n",
    "features_with_route_and_numeric.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Combine `Route_vec` with `NumericFeatures_vec`\n",
    "Now we need to combine our new `Route_vec` with our numeric features. \n",
    "\n",
    "First we create and save the new `VectorAssembler`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created and stored VectorAssembler...\n"
     ]
    }
   ],
   "source": [
    "# Create and save the VectorAssembler\n",
    "route_feature_columns = [\"Route_vec\", \"NumericFeatures_vec\"]\n",
    "route_final_assembler = VectorAssembler(\n",
    "    inputCols=route_feature_columns,\n",
    "    outputCol=\"Features_vec\"\n",
    ")\n",
    "route_final_assembler_path = \"{}/models/final_vector_assembler_simplified_with_route.bin\".format(base_path)\n",
    "route_final_assembler.write().overwrite().save(route_final_assembler_path)\n",
    "\n",
    "print(\"Created and stored VectorAssembler...\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the `VectorAssembler` via `transform` and check it out."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-------------------+-------------------+--------------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|  Route|          Route_vec|NumericFeatures_vec|        Features_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-------------------+-------------------+--------------------+\n",
      "|    32.0|2015-01-01 04:35:...|2015-01-01 03:10:...|     AA|         1|        4|        1|    -1.0| HDN|   769.0|2014-12-31|       35|   DFW|           3.0|DFW-HDN|(4657,[3987],[1.0])|       [-1.0,769.0]|(4659,[3987,4657,...|\n",
      "|   -10.0|2015-01-01 05:20:...|2015-01-01 03:55:...|     AA|         1|        4|        1|    -5.0| DFW|   328.0|2014-12-31|     2377|   ICT|           1.0|ICT-DFW|(4657,[2142],[1.0])|       [-5.0,328.0]|(4659,[2142,4657,...|\n",
      "|    -8.0|2015-01-01 05:00:...|2015-01-01 01:45:...|     AA|         1|        4|        1|    -6.0| MIA|  1096.0|2014-12-31|     1084|   LGA|           1.0|LGA-MIA|  (4657,[64],[1.0])|      [-6.0,1096.0]|(4659,[64,4657,46...|\n",
      "|    52.0|2015-01-01 06:45:...|2015-01-01 00:40:...|     AA|         1|        4|        1|    54.0| ORD|  1671.0|2014-12-31|     1033|   RNO|           3.0|RNO-ORD|(4657,[3108],[1.0])|      [54.0,1671.0]|(4659,[3108,4657,...|\n",
      "|    -6.0|2015-01-01 14:18:...|2015-01-01 10:50:...|     B6|         1|        4|        1|    -2.0| SJU|  1046.0|2014-12-31|     1253|   FLL|           1.0|FLL-SJU| (4657,[435],[1.0])|      [-2.0,1046.0]|(4659,[435,4657,4...|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-------------------+-------------------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "final_vectorized_features_with_route = route_final_assembler.transform(features_with_route_and_numeric)\n",
    "final_vectorized_features_with_route.sample(False, 0.001, 99).show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And drop the intermediate feature vector columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+--------------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|  Route|        Features_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+--------------------+\n",
      "|    -2.0|2015-01-01 06:19:...|2015-01-01 04:35:...|     B6|         1|        4|        1|    -1.0| IAD|   413.0|2014-12-31|      457|   BOS|           1.0|BOS-IAD|(4659,[774,4657,4...|\n",
      "|   -34.0|2015-01-01 11:38:...|2015-01-01 08:10:...|     B6|         1|        4|        1|    -3.0| LAX|  2475.0|2014-12-31|      423|   JFK|           0.0|JFK-LAX|(4659,[2,4657,465...|\n",
      "|   -11.0|2015-01-01 09:54:...|2015-01-01 09:30:...|     DL|         1|        4|        1|    -1.0| MEM|   332.0|2014-12-31|     1587|   ATL|           1.0|ATL-MEM|(4659,[354,4657,4...|\n",
      "|    -4.0|2015-01-01 13:02:...|2015-01-01 10:15:...|     DL|         1|        4|        1|    25.0| ATL|   760.0|2014-12-31|     2510|   JFK|           1.0|JFK-ATL|(4659,[855,4657,4...|\n",
      "|    -5.0|2015-01-01 00:50:...|2014-12-31 23:30:...|     EV|         1|        4|        1|    -6.0| ORD|   632.0|2014-12-31|     5948|   AVP|           1.0|AVP-ORD|(4659,[2836,4657,...|\n",
      "|     3.0|2015-01-01 03:26:...|2015-01-01 02:11:...|     EV|         1|        4|        1|     7.0| IAD|   212.0|2014-12-31|     5976|   EWR|           2.0|EWR-IAD|(4659,[2398,4657,...|\n",
      "|   -11.0|2015-01-01 06:11:...|2015-01-01 03:40:...|     EV|         1|        4|        1|    -4.0| ORD|   794.0|2014-12-31|     4103|   PNS|           1.0|PNS-ORD|(4659,[3740,4657,...|\n",
      "|   -12.0|2015-01-01 04:55:...|2015-01-01 03:20:...|     MQ|         1|        4|        1|    -3.0| ORD|   438.0|2014-12-31|     3010|   SGF|           1.0|SGF-ORD|(4659,[840,4657,4...|\n",
      "|    18.0|2015-01-01 11:00:...|2015-01-01 08:10:...|     NK|         1|        4|        1|    27.0| LGA|  1076.0|2014-12-31|      174|   FLL|           2.0|FLL-LGA|(4659,[111,4657,4...|\n",
      "|   -18.0|2015-01-01 07:26:...|2015-01-01 06:20:...|     OO|         1|        4|        1|   -10.0| SFO|   190.0|2014-12-31|     5385|   SBP|           0.0|SBP-SFO|(4659,[1652,4657,...|\n",
      "|    63.0|2015-01-01 15:19:...|2015-01-01 12:00:...|     UA|         1|        4|        1|    21.0| ORD|   888.0|2014-12-31|      354|   DEN|           3.0|DEN-ORD|(4659,[97,4657,46...|\n",
      "|    14.0|2015-01-01 10:25:...|2015-01-01 07:35:...|     UA|         1|        4|        1|     9.0| EWR|   937.0|2014-12-31|     1100|   MCO|           2.0|MCO-EWR|(4659,[145,4657,4...|\n",
      "|   -12.0|2015-01-01 04:52:...|2015-01-01 03:30:...|     US|         1|        4|        1|   -11.0| BOS|   399.0|2014-12-31|     2112|   DCA|           1.0|DCA-BOS|(4659,[24,4657,46...|\n",
      "|   115.0|2015-01-02 12:30:...|2015-01-02 10:35:...|     AA|         2|        5|        2|   114.0| DFW|   985.0|2015-01-01|     1252|   MCO|           3.0|MCO-DFW|(4659,[250,4657,4...|\n",
      "|    -8.0|2015-01-02 06:25:...|2015-01-02 01:35:...|     AA|         2|        5|        2|    -5.0| DFW|  1171.0|2015-01-01|       36|   SAN|           1.0|SAN-DFW|(4659,[321,4657,4...|\n",
      "|    21.0|2015-01-02 11:44:...|2015-01-02 10:30:...|     DL|         2|        5|        2|    18.0| ATL|   214.0|2015-01-01|     1475|   SAV|           2.0|SAV-ATL|(4659,[340,4657,4...|\n",
      "|     0.0|2015-01-02 09:10:...|2015-01-02 07:00:...|     EV|         2|        5|        2|     9.0| ORD|   584.0|2015-01-01|     6132|   BHM|           2.0|BHM-ORD|(4659,[1672,4657,...|\n",
      "|   -29.0|2015-01-02 12:17:...|2015-01-02 12:08:...|     OO|         2|        5|        2|   -20.0| ORD|   157.0|2015-01-01|     5322|   FWA|           0.0|FWA-ORD|(4659,[671,4657,4...|\n",
      "|     3.0|2015-01-02 07:21:...|2015-01-02 06:35:...|     OO|         2|        5|        2|     4.0| FLG|   119.0|2015-01-01|     6584|   PHX|           2.0|PHX-FLG|(4659,[1028,4657,...|\n",
      "|     6.0|2015-01-02 01:24:...|2015-01-02 00:25:...|     UA|         2|        5|        2|     0.0| IAH|   140.0|2015-01-01|     1014|   AUS|           2.0|AUS-IAH|(4659,[729,4658],...|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for column in route_feature_columns:\n",
    "  final_vectorized_features_with_route = final_vectorized_features_with_route.drop(column)\n",
    "\n",
    "final_vectorized_features_with_route.sample(False, 0.001, 12).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a New Random Forest Classifier Model\n",
    "We need a new model that is fitted with our new feature vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Instantiate and fit random forest classifier on all the data\n",
    "from pyspark.ml.classification import RandomForestClassifier, RandomForestClassificationModel\n",
    "rfc = RandomForestClassifier(featuresCol=\"Features_vec\", labelCol=\"ArrDelayBucket\", predictionCol=\"Prediction\")\n",
    "model = rfc.fit(final_vectorized_features)\n",
    "# Save the new model over the old one\n",
    "model_output_path = \"{}/models/spark_random_forest_classifier.flight_delays_simplified_with_route.bin\".format(\n",
    "  base_path\n",
    ")\n",
    "model.write().overwrite().save(model_output_path)\n",
    "print(\"Model trained on 5.7MM items successfully!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+--------------------+--------------------+----------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|        Features_vec|       rawPrediction|         probability|Prediction|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+--------------------+--------------------+----------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|(646,[53,324,644,...|[3.66319242779629...|[0.18315962138981...|       1.0|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|(646,[53,324,644,...|[3.66319242779629...|[0.18315962138981...|       1.0|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|(646,[53,324,644,...|[3.85744835938781...|[0.19287241796939...|       1.0|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|    16.0|2015-01-01 07:15:...|2015-01-01 05:45:...|     AA|         1|        4|        1|    15.0| DFW|   731.0|2014-12-31|     1473|   ATL|           2.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|    -7.0|2015-01-01 04:15:...|2015-01-01 02:45:...|     AA|         1|        4|        1|    -2.0| DFW|   731.0|2014-12-31|     1513|   ATL|           1.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|    13.0|2015-01-01 08:50:...|2015-01-01 07:25:...|     AA|         1|        4|        1|     9.0| DFW|   731.0|2014-12-31|      194|   ATL|           2.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|    25.0|2015-01-01 12:30:...|2015-01-01 11:00:...|     AA|         1|        4|        1|    -2.0| DFW|   731.0|2014-12-31|      232|   ATL|           2.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|    58.0|2015-01-01 13:40:...|2015-01-01 12:15:...|     AA|         1|        4|        1|    14.0| DFW|   731.0|2014-12-31|      276|   ATL|           3.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|    14.0|2015-01-01 05:25:...|2015-01-01 03:55:...|     AA|         1|        4|        1|    15.0| DFW|   731.0|2014-12-31|      314|   ATL|           2.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|     1.0|2015-01-01 10:05:...|2015-01-01 08:40:...|     AA|         1|        4|        1|    -5.0| DFW|   731.0|2014-12-31|      356|   ATL|           2.0|(646,[0,324,644,6...|[4.16869700315495...|[0.20843485015774...|       1.0|\n",
      "|   -29.0|2015-01-01 02:12:...|2015-01-01 00:15:...|     AA|         1|        4|        1|    -9.0| MIA|   594.0|2014-12-31|     1652|   ATL|           0.0|(646,[0,346,644,6...|[3.68497926035629...|[0.18424896301781...|       1.0|\n",
      "|   -10.0|2015-01-01 00:52:...|2014-12-31 23:00:...|     AA|         1|        4|        1|    -4.0| MIA|   594.0|2014-12-31|       17|   ATL|           1.0|(646,[0,346,644,6...|[3.68497926035629...|[0.18424896301781...|       1.0|\n",
      "|    -3.0|2015-01-01 15:02:...|2015-01-01 13:10:...|     AA|         1|        4|        1|    -7.0| MIA|   594.0|2014-12-31|      349|   ATL|           1.0|(646,[0,346,644,6...|[3.68497926035629...|[0.18424896301781...|       1.0|\n",
      "|    -8.0|2015-01-01 06:35:...|2015-01-01 05:30:...|     AA|         1|        4|        1|    -2.0| DFW|   190.0|2014-12-31|     1023|   AUS|           1.0|(646,[34,324,644,...|[3.78344522743971...|[0.18917226137198...|       1.0|\n",
      "|    -1.0|2014-12-31 22:50:...|2014-12-31 21:50:...|     AA|         1|        4|        1|    -2.0| DFW|   190.0|2014-12-31|     1178|   AUS|           1.0|(646,[34,324,644,...|[3.78344522743971...|[0.18917226137198...|       1.0|\n",
      "|   -14.0|2015-01-01 01:40:...|2015-01-01 00:30:...|     AA|         1|        4|        1|    -6.0| DFW|   190.0|2014-12-31|     1296|   AUS|           1.0|(646,[34,324,644,...|[3.78344522743971...|[0.18917226137198...|       1.0|\n",
      "|   -16.0|2015-01-01 02:15:...|2015-01-01 01:05:...|     AA|         1|        4|        1|    -4.0| DFW|   190.0|2014-12-31|     1356|   AUS|           0.0|(646,[34,324,644,...|[3.78344522743971...|[0.18917226137198...|       1.0|\n",
      "|    18.0|2015-01-01 08:55:...|2015-01-01 07:55:...|     AA|         1|        4|        1|     3.0| DFW|   190.0|2014-12-31|     1365|   AUS|           2.0|(646,[34,324,644,...|[3.78344522743971...|[0.18917226137198...|       1.0|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+--------------------+--------------------+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n",
      "+----------+-------+\n",
      "|Prediction|  count|\n",
      "+----------+-------+\n",
      "|       1.0|5713778|\n",
      "|       3.0|    230|\n",
      "+----------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = model.transform(final_vectorized_features)\n",
    "predictions.groupBy(\"Prediction\").count().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try Just StringIndexer\n",
    "The internet says not to use `OneHotEncoder`, so I'll try that."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-----------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|  Route|Route_index|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-----------+\n",
      "|    13.0|2015-01-01 10:10:...|2015-01-01 07:30:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1024|   ABQ|           2.0|ABQ-DFW|      938.0|\n",
      "|    17.0|2015-01-01 02:15:...|2014-12-31 23:25:...|     AA|         1|        4|        1|    14.0| DFW|   569.0|2014-12-31|     1184|   ABQ|           2.0|ABQ-DFW|      938.0|\n",
      "|    36.0|2015-01-01 03:45:...|2015-01-01 01:00:...|     AA|         1|        4|        1|    -2.0| DFW|   569.0|2014-12-31|      336|   ABQ|           3.0|ABQ-DFW|      938.0|\n",
      "|   -21.0|2015-01-01 11:30:...|2015-01-01 09:55:...|     AA|         1|        4|        1|    -1.0| DFW|   731.0|2014-12-31|      125|   ATL|           0.0|ATL-DFW|       37.0|\n",
      "|   -14.0|2015-01-01 02:25:...|2015-01-01 00:55:...|     AA|         1|        4|        1|    -4.0| DFW|   731.0|2014-12-31|     1455|   ATL|           1.0|ATL-DFW|       37.0|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-----------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.feature import StringIndexer, OneHotEncoder, VectorIndexer\n",
    "\n",
    "route_string_indexer = StringIndexer(\n",
    "  inputCol='Route',\n",
    "  outputCol='Route' + \"_index\"\n",
    ")\n",
    "route_string_indexer_model = route_string_indexer.fit(features_with_route)\n",
    "\n",
    "# Fit/transform to apply to the data\n",
    "features_with_route_index = route_string_indexer_model.transform(features_with_route)\n",
    "features_with_route_index.show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now compose the `Route_index` with the `DepDelay` and `Distance` columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Handle continuous, numeric fields by combining them into one feature vector\n",
    "new_numeric_columns = [\"DepDelay\", \"Distance\", \"Route_index\"]\n",
    "route_index_vector_assembler = VectorAssembler(\n",
    "  inputCols=new_numeric_columns,\n",
    "  outputCol=\"Features_vec\"\n",
    ")\n",
    "final_route_index_features = route_index_vector_assembler.transform(features_with_route_index)\n",
    "\n",
    "# Save the numeric vector assembler\n",
    "route_index_vector_assembler_path = \"{}/models/numeric_vector_assembler_route_index.bin\".format(base_path)\n",
    "route_index_vector_assembler.write().overwrite().save(route_index_vector_assembler_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-----------+--------------------+\n",
      "|ArrDelay|          CRSArrTime|          CRSDepTime|Carrier|DayOfMonth|DayOfWeek|DayOfYear|DepDelay|Dest|Distance|FlightDate|FlightNum|Origin|ArrDelayBucket|  Route|Route_index|        Features_vec|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-----------+--------------------+\n",
      "|   -19.0|2015-01-01 13:00:...|2015-01-01 04:45:...|     AA|         1|        4|        1|     0.0| BOS|  2611.0|2014-12-31|      202|   LAX|           0.0|LAX-BOS|      206.0|  [0.0,2611.0,206.0]|\n",
      "|    -7.0|2015-01-01 07:14:...|2015-01-01 05:30:...|     AS|         1|        4|        1|    -6.0| SEA|   605.0|2014-12-31|      359|   SMF|           1.0|SMF-SEA|      280.0|  [-6.0,605.0,280.0]|\n",
      "|     0.0|2015-01-01 10:18:...|2015-01-01 07:55:...|     B6|         1|        4|        1|     7.0| DTW|   632.0|2014-12-31|     1237|   BOS|           2.0|BOS-DTW|      299.0|   [7.0,632.0,299.0]|\n",
      "|   -27.0|2015-01-01 03:30:...|2014-12-31 23:45:...|     B6|         1|        4|        1|   -10.0| SJU|  1189.0|2014-12-31|     1033|   MCO|           0.0|MCO-SJU|      281.0|[-10.0,1189.0,281.0]|\n",
      "|   -15.0|2015-01-01 06:29:...|2015-01-01 04:12:...|     DL|         1|        4|        1|     3.0| JFK|   760.0|2014-12-31|     1218|   ATL|           1.0|ATL-JFK|      852.0|   [3.0,760.0,852.0]|\n",
      "+--------+--------------------+--------------------+-------+----------+---------+---------+--------+----+--------+----------+---------+------+--------------+-------+-----------+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "final_route_index_features.sample(False,0.001,88).show(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train a new model on this new set of features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model trained on 5.7MM items successfully!\n"
     ]
    }
   ],
   "source": [
    "# Instantiate and fit random forest classifier on all the data\n",
    "from pyspark.ml.classification import RandomForestClassifier, RandomForestClassificationModel\n",
    "rfc = RandomForestClassifier(featuresCol=\"Features_vec\", labelCol=\"ArrDelayBucket\", predictionCol=\"Prediction\", maxBins=4657)\n",
    "model = rfc.fit(final_route_index_features)\n",
    "# Save the new model over the old one\n",
    "model_output_path = \"{}/models/spark_random_forest_classifier.flight_delays_simplified_with_route_index.bin\".format(\n",
    "  base_path\n",
    ")\n",
    "model.write().overwrite().save(model_output_path)\n",
    "print(\"Model trained on 5.7MM items successfully!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And make predictions on the training data and inspect their distribution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+-------+\n",
      "|Prediction|  count|\n",
      "+----------+-------+\n",
      "|       0.0|   1165|\n",
      "|       1.0|4143264|\n",
      "|       3.0| 565640|\n",
      "|       2.0|1003939|\n",
      "+----------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions = model.transform(final_route_index_features)\n",
    "predictions.groupBy(\"Prediction\").count().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It works!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------+-------+\n",
      "|ArrDelayBucket|  count|\n",
      "+--------------+-------+\n",
      "|           0.0|1098212|\n",
      "|           1.0|2402687|\n",
      "|           3.0| 650604|\n",
      "|           2.0|1562505|\n",
      "+--------------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predictions.groupBy(\"ArrDelayBucket\").count().show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Solution! We Sovled the Problem!\n",
    "\n",
    "We were mistaken in using `OneHotEncoder` on the values of `StringIndexerModel`, we should have just used `VectorAssembler` directly on the string indexes. After changing the `maxBins` parameter of our `RandomForestClassifier`, it works!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}